PROJECT 058 · PYTHON SOURCE

Sensor association drift

Study-specific code. Shared modules, dependency versions, and reproduction instructions are included in all project files.

from pathlib import Path
import sys
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT))
import numpy as np
import pandas as pd
from scipy.stats import spearmanr
from sklearn.decomposition import PCA
from sklearn.ensemble import IsolationForest
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import confusion_matrix, precision_score, recall_score
from portfolio.data import load
from portfolio.constants import *
from portfolio.methods import result, sql, rates, associations, distribution, regression, classification, cluster
from portfolio.engine import execute

META = {'id': 58, 'dataset': 'air', 'title': 'Sensor association drift', 'question': 'How stable is the relationship between a CO-sensitive sensor and the reference over calendar months?', 'method': 'Monthly paired correlation, coverage and median level comparison.', 'action': 'Investigate changes in calibration conditions before updating a monitoring model.', 'limitations': 'The -200 sentinel is treated as missing. Reference instruments have incomplete coverage; comparisons use paired observations. Sensor calibration is retrospective, not a health or regulatory compliance assessment. '}

def analyze():
    df = load('air').copy()
    rows=[]
    for month,g in df.groupby('month'):
        pair=g[['co_gt','pt08_s1_co']].dropna()
        if len(pair)>=30:
            rows.append({'month':month,'paired_hours':len(pair),'rho':pair.corr(method='spearman').iloc[0,1],'median_co':pair.co_gt.median(),'median_sensor':pair.pt08_s1_co.median()})
    t=pd.DataFrame(rows)
    out=result(t,'month','rho','Spearman correlation','A changing correlation can reflect pollution range or environmental shifts, not necessarily hardware deterioration.',kind='line')
    return out

if __name__ == "__main__":
    execute(Path(__file__).parent, META, analyze)