PROJECT 004 · PYTHON SOURCE

Market basket associations

Study-specific code. Shared modules, dependency versions, and reproduction instructions are included in all project files.

from pathlib import Path
import sys
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT))
import numpy as np
import pandas as pd
from scipy.stats import spearmanr
from sklearn.decomposition import PCA
from sklearn.ensemble import IsolationForest
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import confusion_matrix, precision_score, recall_score
from portfolio.data import load
from portfolio.constants import *
from portfolio.methods import result, sql, rates, associations, distribution, regression, classification, cluster
from portfolio.engine import execute

META = {'id': 4, 'dataset': 'retail', 'title': 'Market basket associations', 'question': 'Which frequently purchased product pairs co-occur more than their individual popularity suggests?', 'method': 'Invoice presence matrix and pair support, confidence and lift among the 30 most frequent codes.', 'action': 'Use a small merchandising experiment to test a selected pair; co-purchase alone does not establish an uplift.', 'limitations': 'Historical invoice lines; credits are not reliably matched to original sales. Gross purchases are not profit. Unidentified customers cannot support customer-level conclusions. Exact repeated lines remain unless the study explicitly compares removal. '}

def analyze():
    df = load('retail').copy()
    s=df[df.is_sale]
    top=s.groupby('stockcode').invoiceno.nunique().nlargest(30).index
    presence=pd.crosstab(s[s.stockcode.isin(top)].invoiceno,s[s.stockcode.isin(top)].stockcode).gt(0).astype(int)
    n=s.invoiceno.nunique()
    counts=presence.sum(); pairs=presence.T.dot(presence)
    rows=[]
    for i,a in enumerate(presence.columns):
        for b in presence.columns[i+1:]:
            both=int(pairs.loc[a,b])
            if both>=50:
                rows.append({'pair':a+' + '+b,'co_invoices':both,'all_sale_invoices':n,'support':both/n,'confidence_a_to_b':both/counts[a],'lift':both*n/(counts[a]*counts[b])})
    t=pd.DataFrame(rows).sort_values('lift',ascending=False)
    out=result(t,'pair','lift','Lift','Product search is restricted to 30 popular codes; require at least 50 co-purchase invoices. The denominator includes all positive-sales invoices.')
    return out

if __name__ == "__main__":
    execute(Path(__file__).parent, META, analyze)