PROJECT 053 · PYTHON SOURCE

Air monitoring data availability

Study-specific code. Shared modules, dependency versions, and reproduction instructions are included in all project files.

from pathlib import Path
import sys
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT))
import numpy as np
import pandas as pd
from scipy.stats import spearmanr
from sklearn.decomposition import PCA
from sklearn.ensemble import IsolationForest
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import confusion_matrix, precision_score, recall_score
from portfolio.data import load
from portfolio.constants import *
from portfolio.methods import result, sql, rates, associations, distribution, regression, classification, cluster
from portfolio.engine import execute

META = {'id': 53, 'dataset': 'air', 'title': 'Air monitoring data availability', 'question': 'Which sensor and reference fields have the weakest observation coverage?', 'method': 'Field-level availability after converting the documented -200 missing sentinel.', 'action': 'Prioritize recovery of reference measurements before calibrating sensors.', 'limitations': 'The -200 sentinel is treated as missing. Reference instruments have incomplete coverage; comparisons use paired observations. Sensor calibration is retrospective, not a health or regulatory compliance assessment. '}

def analyze():
    df = load('air').copy()
    features=['co_gt','nmhc_gt','c6h6_gt','nox_gt','no2_gt',*AIR_SENSORS]
    t=pd.DataFrame([{'field':c,'observed':int(df[c].notna().sum()),'missing':int(df[c].isna().sum()),'missing_share':df[c].isna().mean()} for c in features]).sort_values('missing_share',ascending=False)
    out=result(t,'field','missing_share','Missing share','Entirely blank CSV rows and unnamed empty columns were removed; missing instrument readings are retained as NaN.')
    return out

if __name__ == "__main__":
    execute(Path(__file__).parent, META, analyze)