PROJECT 006 · PYTHON SOURCE

Sales data cleaning sensitivity

Study-specific code. Shared modules, dependency versions, and reproduction instructions are included in all project files.

from pathlib import Path
import sys
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT))
import numpy as np
import pandas as pd
from scipy.stats import spearmanr
from sklearn.decomposition import PCA
from sklearn.ensemble import IsolationForest
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import confusion_matrix, precision_score, recall_score
from portfolio.data import load
from portfolio.constants import *
from portfolio.methods import result, sql, rates, associations, distribution, regression, classification, cluster
from portfolio.engine import execute

META = {'id': 6, 'dataset': 'retail', 'title': 'Sales data cleaning sensitivity', 'question': 'How do plausible cleaning choices alter gross invoiced sales?', 'method': 'Compare retained source rows, exact-row deduplication and exclusion of unidentified purchasers.', 'action': 'Choose cleaning rules from source-system evidence and document their financial effect.', 'limitations': 'Historical invoice lines; credits are not reliably matched to original sales. Gross purchases are not profit. Unidentified customers cannot support customer-level conclusions. Exact repeated lines remain unless the study explicitly compares removal. '}

def analyze():
    df = load('retail').copy()
    s=df[df.is_sale]
    original=[c for c in df if c not in ['row_id','value','is_sale','month']]
    rows=[{'policy':'Retain source repeats','rows':len(s),'gross_gbp':s.value.sum()},
     {'policy':'Remove exact repeats','rows':len(s.drop_duplicates(original)),'gross_gbp':s.drop_duplicates(original).value.sum()},
     {'policy':'Known customers only','rows':int(s.customerid.notna().sum()),'gross_gbp':s.loc[s.customerid.notna(),'value'].sum()}]
    t=pd.DataFrame(rows);t['difference_from_retained_gbp']=t.gross_gbp-t.gross_gbp.iloc[0]
    out=result(t,'policy','gross_gbp','GBP','Unknown customers and repeated lines are different quality issues; neither is automatically an invalid sale.')
    return out

if __name__ == "__main__":
    execute(Path(__file__).parent, META, analyze)