Impute pipelines

from random import randint

import pandas as pd
import numpy as np

from sklearn.preprocessing import OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, ExtraTreesRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.metrics import mean_squared_error, median_absolute_error

from hyperopt import fmin, tpe, hp, Trials, STATUS_OK
import mlflow

import matplotlib.pyplot as plt
import seaborn as sns
sns.set()

def generate_array_with_random_nan(lower_bound, upper_bound, size):
    a = np.random.randint(lower_bound, upper_bound+1, size=size).astype(float)
    mask = np.random.choice([1, 0], a.shape, p=[.1, .9]).astype(bool)
    a[mask] = np.nan

    return a

size = 6000

df_cbd = pd.DataFrame()
df_cbd['bed'] = generate_array_with_random_nan(1, 2, size)
df_cbd['bath'] = generate_array_with_random_nan(1, 2, size)
df_cbd['area_usable'] = np.random.randint(20, 40, size=size)
df_cbd['region'] = 'cbd'

df_suburb = pd.DataFrame()
df_suburb['bed'] = generate_array_with_random_nan(1, 4, size)
df_suburb['bath'] = generate_array_with_random_nan(1, 4, size)
df_suburb['area_usable'] = np.random.randint(30, 200, size=size)
df_suburb['region'] = 'suburb'

df = pd.concat([df_cbd, df_suburb])
df

def report_missing(df):
    cnts = []
    cnt_total = len(df)
    for col in df.columns:
        cnt_missing = sum(pd.isnull(df[col]) | pd.isna(df[col]))
        print("col: {}, missing: {}%".format(col, 100.0 * cnt_missing / cnt_total))

        cnts.append({
            'column': col,
            'missing': 100.0 * cnt_missing / cnt_total
        })

    cnts_df = pd.DataFrame(cnts)
    sns.barplot(x=cnts_df.missing,
                y=cnts_df.column,
    #             palette=['r','b'],
    #             data=cnts_df
                )

    return sns

report_missing(df)

## missing bed per region
df[df.bed.isna()]["region"].value_counts(dropna=False)

## missing bath per region
df[df.bath.isna()]["region"].value_counts(dropna=False)

## explore region
df.region.value_counts()

## explore bed
df.bed.value_counts()

## explore bath
df.bath.value_counts()

## remove outliers here

synth_columns = {
    'bed': {
        "region_bath": ['region', 'bath']
    },
    'bath': {
        "region_bed": ['region', 'bed']
    }
}

for column, groupby_levels in synth_columns.items():
    for groupby_level_name, groupby_columns in groupby_levels.items():
        # percentile aggregates
        for pctl in [20,50,80,90]:
            col_name = 'p{}|{}|{}'.format(pctl, groupby_level_name, column)
            print("calculating -- {}".format(col_name))
            df[col_name] = df[groupby_columns+[column]].fillna(0).groupby(groupby_columns)[column].transform(lambda x: x.quantile(pctl/100.0))

        # mean impute
        mean_impute = 'mean|{}|{}'.format(groupby_level_name,column)
        print("calculating -- {}".format(mean_impute))
        df[mean_impute] = df.groupby(groupby_columns)[column].transform('mean')

        # bed/bath rank
        rank_impute = column_name = 'rank|{}|{}'.format(groupby_level_name,column)
        print("calculating -- {}".format(rank_impute))
        df[rank_impute] = df.groupby(groupby_columns)[column].rank(method='dense', na_option='bottom')

def coalesce(df, columns):
    '''
    Implement coalesce of function in colunms.

    Inputs:
    df: reference dataframe
    columns: columns to perform coalesce

    Returns:
    df_tmp: pd.Series that is coalesced

    Example:
    df_tmp = pd.DataFrame({'a': [1,2,None,None,None,None],
                            'b': [None,6,None,8,9,None],
                            'c': [None,10,None,12,None,13]})
    df_tmp['new'] = coalesce(df_tmp, ['a','b','c'])
    print(df_tmp)
    '''
    df_tmp = df[columns[0]]
    for c in columns[1:]:
        df_tmp = df_tmp.fillna(df[c])

    return df_tmp


coalesce_columns = [
    'bed',
    'p50|region_bath|bed',
    # p50|GROUPBY_LESSER_WEIGHT|bed, ...
]

df["bed_imputed"] = coalesce(df, coalesce_columns)

coalesce_columns = [
    'bath',
    'p50|region_bed|bath',
     # p50|GROUPBY_LESSER_WEIGHT|bath, ...
]

df["bath_imputed"] = coalesce(df, coalesce_columns)

report_missing(df)

## assign partition
def assign_partition(x):
    if x in [0,1,2,3,4,5]:
        return 0
    elif x in [6,7]:
        return 1
    else:
        return 2

## assign random id
df['listing_id'] = [randint(1000000, 9999999) for i in range(len(df))]

## hashing
df["hash_id"] = df["listing_id"].apply(lambda x: x % 10)

## assign partition
df["partition_id"] = df["hash_id"].apply(lambda x: assign_partition(x))

## define columns group
y_column = 'area_usable'

categ_columns = ['region']

numer_columns = [
    'bed_imputed',
    'bath_imputed',

    'p20|region_bath|bed',
    'p50|region_bath|bed',
    'p80|region_bath|bed',
    'p90|region_bath|bed',
    'mean|region_bath|bed',
    'rank|region_bath|bed',

    'p20|region_bed|bath',
    'p50|region_bed|bath',
    'p80|region_bed|bath',
    'p90|region_bed|bath',
    'mean|region_bed|bath',
    'rank|region_bed|bath',
]

id_columns = [
    'listing_id',
    'hash_id',
    'partition_id'
]

## remove missing y
df = df.dropna(subset=[y_column])

## split into train-dev-test
df_train = df[df["partition_id"] == 0]
df_dev = df[df["partition_id"] == 1]
df_test = df[df["partition_id"] == 2]

## split each set into x and y
y_train = df_train[y_column].values
df_train = df_train[numer_columns+categ_columns]

y_dev = df_dev[y_column].values
df_dev = df_dev[numer_columns+categ_columns]

y_test = df_test[y_column].values
df_test = df_test[numer_columns+categ_columns]

## define pipelines
impute_median = SimpleImputer(strategy='median')
impute_mode = SimpleImputer(strategy='most_frequent')

num_pipeline = Pipeline([
        ('impute_median', impute_median),
        ('std_scaler', StandardScaler()),
    ])

categ_pipeline = Pipeline([
        ('impute_mode', impute_mode),
        ('categ_1hot', OneHotEncoder(handle_unknown='ignore')),
    ])

full_pipeline = ColumnTransformer([
        ("num", num_pipeline, numer_columns),
        ("cat", categ_pipeline, categ_columns),
    ])

## fit and transform
X_train = full_pipeline.fit_transform(df_train)
X_dev = full_pipeline.transform(df_dev)
X_test = full_pipeline.transform(df_test)

X_train

## mlflow + hyperopt combo
def objective(params):
    regressor_type = params['type']
    del params['type']
    if regressor_type == 'gradient_boosting_regression':
        estimator = GradientBoostingRegressor(**params)
    elif regressor_type == 'random_forest_regression':
        estimator = RandomForestRegressor(**params)
    elif regressor_type == 'extra_trees_regression':
        estimator = ExtraTreesRegressor(**params)
    elif regressor_type == 'decision_tree_regression':
        estimator = DecisionTreeRegressor(**params)
    else:
        return 0

    estimator.fit(X_train, y_train)

    # mae
    y_dev_hat = estimator.predict(X_dev)
    mae = median_absolute_error(y_dev, y_dev_hat)

    # logging
    with mlflow.start_run():
        mlflow.log_param("regressor", estimator.__class__.__name__)
        # mlflow.log_param("params", params)
        mlflow.log_param('n_estimators', params.get('n_estimators'))
        mlflow.log_param('max_depth', params.get('max_depth'))

        mlflow.log_metric("median_absolute_error", mae)

    return {'loss': mae, 'status': STATUS_OK}

space = hp.choice('regressor_type', [
    {
        'type': 'gradient_boosting_regression',
        'n_estimators': hp.choice('n_estimators1', range(100,200,50)),
        'max_depth': hp.choice('max_depth1', range(10,13,1))
    },
    {
        'type': 'random_forest_regression',
        'n_estimators': hp.choice('n_estimators2', range(100,200,50)),
        'max_depth': hp.choice('max_depth2', range(3,25,1)),
        'n_jobs': -1

    },
    {
        'type': 'extra_trees_regression',
        'n_estimators': hp.choice('n_estimators3', range(100,200,50)),
        'max_depth': hp.choice('max_depth3', range(3,10,2))
    },
    {
        'type': 'decision_tree_regression',
        'max_depth': hp.choice('max_depth4', range(3,10,2))
    }
])

trials = Trials()
max_evals = 40

best = fmin(
fn=objective,
space=space,
algo=tpe.suggest,
max_evals=max_evals,
trials=trials)

print("Found minimum after {} trials:".format(max_evals))
from pprint import pprint
pprint(best)

## use best params on TEST set
estimator = RandomForestRegressor(max_depth=4, n_estimators=150)
estimator.fit(X_train, y_train)

y_train_hat = estimator.predict(X_train)
train_mae = median_absolute_error(y_train, y_train_hat)

y_dev_hat = estimator.predict(X_dev)
dev_mae = median_absolute_error(y_dev, y_dev_hat)

y_test_hat = estimator.predict(X_test)
test_mae = median_absolute_error(y_test, y_test_hat)

mae =  {
    'name': estimator.__class__.__name__,
    'train_mae': train_mae,
    'dev_mae': dev_mae,
    'test_mae': test_mae
}

mae = pd.DataFrame([mae]).set_index('name')

mae