{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction\nSource: https://www.kaggle.com/competitions/elo-merchant-category-recommendation/data\n\n### Table of Contents\n- [Libraries](#libraries)\n- [Utils](#utils)\n- [Datasets](#datasets)\n- [EDA](#eda)\n- [Imputation](#imputation)\n- [Feature Engineering](#feature-engineering)\n- [Feature Selection](#feature-selection)\n- [Machine Learning](#machine-learning)","metadata":{}},{"cell_type":"markdown","source":"### Libraries <a id=\"libraries\"></a>","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom pandas import DataFrame\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\n# Tools\nimport math\nimport datetime\nfrom typing import List, Union\n\n# ML Tools\nfrom sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV\nfrom sklearn.metrics import mean_squared_error\n\n# Regression Models\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.tree import DecisionTreeRegressor\nimport lightgbm as lgb\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.feature_selection import RFE\n\n# CONSTANTS\nSEED = 123\nTEST_PERC = 0.05\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport unittest\nimport threading","metadata":{"execution":{"iopub.status.busy":"2023-03-27T01:12:53.395752Z","iopub.execute_input":"2023-03-27T01:12:53.396972Z","iopub.status.idle":"2023-03-27T01:12:55.906494Z","shell.execute_reply.started":"2023-03-27T01:12:53.396852Z","shell.execute_reply":"2023-03-27T01:12:55.90494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Utilts <a id=\"utils\"></a>","metadata":{}},{"cell_type":"code","source":"def summarizeDF(df:DataFrame)->DataFrame:\n    \"\"\"This function shows a basic summary of the given dataframe.\n    \n    Parameters\n    ----------\n    df: pandas DataFrame\n    This specifies the dataframe to be summarized.\n    \n    Returns\n    -------\n    pandas DataFrame: This is a table of summary of the given dataset.\n    \"\"\"    \n    variables, dtypes, count, unique, missing, pc_missing = [], [], [], [], [], []\n    \n    for item in df.columns:\n        variables.append(item)\n        dtypes.append(df[item].dtype)\n        count.append(len(df[item]))\n        unique.append(len(df[item].unique()))\n        missing.append(df[item].isna().sum())\n        pc_missing.append(round((df[item].isna().sum() / len(df[item])) * 100, 2))\n\n    output = pd.DataFrame({\n        'column_name': variables, \n        'dtype': dtypes,\n        'count': count,\n        'unique': unique,\n        'missing': missing, \n        'percentage_missing_data': pc_missing\n    })    \n        \n    return output\n\ndef preprocess_data(df:DataFrame=None)->DataFrame:\n    \"\"\"This function preprocess the data into a specific form for the computation.\n    Given a DataFrame (df), impute with mode.\n    \n    Parameters\n    ----------\n    df: pandas DataFrame\n    This specifies the data to be preprocessed.\n    \n    Returns\n    -------\n    DataFrame: This specifies the preprocessed DataFrame.\n    \"\"\"\n    if df is None:\n        raise Exception(\"Expected a DataFrame, no DataFrame supplied.\")\n    \n    df_copy = df.copy()\n    for col in df.columns[df.isnull().any()]:\n        df_copy[col].fillna(df_copy['card_id'].map(df_copy.groupby('card_id')[col].apply(lambda x: x.mode().iloc[0] if not x.isnull().all() else np.nan)).fillna(df_copy[col].mode().iloc[0]), inplace=True)\n\n    return df_copy\n\ndef feature_engineering(df:DataFrame=None)->DataFrame:\n    \"\"\"This function perform feature engineering on the input Data\"\"\"\n    \n    def get_new_columns(name:str, aggs:list)->list: # Nested function for feature engineering\n        \"\"\"This function creates new column names for the aggregation of the features.\"\"\"\n        return [name + '_' + k + '_' + agg for k in aggs.keys() for agg in aggs[k]]\n    \n    # Make copy of df\n    df_historical_transactions_copy = df.copy()\n\n    # Convert DT columns to Pandas DT\n    df_historical_transactions_copy['purchase_date'] = pd.to_datetime(df_historical_transactions_copy['purchase_date'])\n\n    # Feature Engineer columns from purchase_date\n    df_historical_transactions_copy['year'] = df_historical_transactions_copy['purchase_date'].dt.year\n    df_historical_transactions_copy['weekofyear'] = df_historical_transactions_copy['purchase_date'].dt.isocalendar().week\n    df_historical_transactions_copy['month'] = df_historical_transactions_copy['purchase_date'].dt.month\n    df_historical_transactions_copy['dayofweek'] = df_historical_transactions_copy['purchase_date'].dt.dayofweek\n    df_historical_transactions_copy['weekend'] = (df_historical_transactions_copy.purchase_date.dt.weekday >=5).astype(int)\n    df_historical_transactions_copy['hour'] = df_historical_transactions_copy['purchase_date'].dt.hour\n\n    # Encode Binary Features\n    df_historical_transactions_copy['authorized_flag'] = df_historical_transactions_copy['authorized_flag'].map({\"Y\":1, 'N':0})\n    df_historical_transactions_copy['category_1'] = df_historical_transactions_copy['category_1'].map({'Y':1, 'N':0})\n\n    # Feature Engineer Month Diff/Lag\n    df_historical_transactions_copy['month_diff'] = ((datetime.datetime.today() - df_historical_transactions_copy['purchase_date']).dt.days)//30\n    df_historical_transactions_copy['month_diff'] += df_historical_transactions_copy['month_lag']\n    \n    # Getting Centrality of the Data\n    aggs = {}\n    for col in ['month','hour','weekofyear','dayofweek','year', 'state_id','subsector_id']:\n        aggs[col] = ['nunique']\n\n    # Feature Engineering using Univariate Analysis\n    aggs['authorized_flag'] = ['sum', 'mean']\n    aggs['card_id'] = ['size']\n    aggs['category_1'] = ['sum', 'mean']\n    aggs['installments'] = ['sum','max','min','mean','var']\n    aggs['month_lag'] = ['max','min','mean','var']\n    aggs['purchase_amount'] = ['sum','max','min','mean','var']\n    aggs['purchase_date'] = ['max','min']\n    aggs['month_diff'] = ['mean']\n    aggs['weekend'] = ['sum', 'mean']\n\n    for col in ['category_2','category_3']:\n        df_historical_transactions_copy[col+'_mean'] = df_historical_transactions_copy.groupby([col])['purchase_amount'].transform('mean')\n        aggs[col+'_mean'] = ['mean']    \n\n    new_columns = get_new_columns('hist',aggs)\n    \n    # Group Aggregations by card_id\n    df_historical_transactions_copy_group = df_historical_transactions_copy.groupby('card_id').agg(aggs)\n\n    # Remove Multilevel Indexing with New Column Names\n    df_historical_transactions_copy_group.columns = new_columns\n    \n    # Reset Index\n    df_historical_transactions_copy_group.reset_index(drop=False,inplace=True)\n    \n    # Cast variable to pandas Datetime\n    df_historical_transactions_copy_group['hist_purchase_date_max'] = pd.to_datetime(df_historical_transactions_copy_group['hist_purchase_date_max'])\n    df_historical_transactions_copy_group['hist_purchase_date_min'] = pd.to_datetime(df_historical_transactions_copy_group['hist_purchase_date_min'])\n    \n    return df_historical_transactions_copy_group\n\ndef merge_data(key:str=None, dfs:List[DataFrame]=None)->DataFrame:\n    \"\"\"This function takes in multiple dataframes and performs a left outer join on a key.\n    \n    Parameters\n    ----------\n    key: str\n    This species the joining key.\n    \n    dfs: list of pandas DataFrame\n    This specifies the list of DataFrames to perform left outer join based on a key.\n    \n    Returns\n    -------\n    pandas DataFrame: This specifies the resultant DataFrame from the merging operation.\n    \"\"\"\n    \n    # Sanity Check\n    if key is None:\n        raise Exception(\"Expected a key, no key supplied.\")\n        \n    if not isinstance(key, str):\n        raise Exception(f\"Expected type str for key, {type(key)} was supplied.\")\n    \n    if dfs is None or not len(dfs) == 2:\n        raise Exception(\"Expected at least two DataFrame.\")\n        \n    if any(type(x) != DataFrame for x in dfs):\n        raise Exception(\"At least one DataFrame is not the correct DataType.\")\n        \n    # Iterate through DataFrames to perform merge operation\n    df_res = dfs[0]\n    \n    for df in dfs[1:]:\n        df_res = pd.merge(left=df_res, right=df, how='outer', left_on=key, right_on=key)\n    \n    return df_res\n\ndef zhenjie_miracle(df_historical_transactions_copy_group):\n    brazil_holiday_list=[ \n            '01-01-17', '14-02-17', '28-08-17', '14-04-17', '16-04-17', '21-04-17',\n            '01-05-17', '15-06-17', '07-09-17', '12-10-17', '02-11-17', '15-11-17', \n            '24-12-17', '25-12-17', '31-12-17',\n            '01-01-18', '14-02-18', '28-08-18', '14-04-18', '16-04-18', '21-04-18',\n            '01-05-18', '15-06-18', '07-09-18', '12-10-18', '02-11-18', '15-11-18', \n            '24-12-18', '25-12-18', '31-12-18'\n      ]\n    df_historical_transactions_copy_group['purchase_max_is_holiday'] = df_historical_transactions_copy_group['hist_purchase_date_max'].isin(brazil_holiday_list).astype(int)\n    df_historical_transactions_copy_group['purchase_min_is_holiday'] = df_historical_transactions_copy_group['hist_purchase_date_min'].isin(brazil_holiday_list).astype(int)\n    \n    df_historical_transactions_copy_group_dummies = pd.get_dummies(df_historical_transactions_copy_group['feature_1'], prefix='feature_1')#, drop_first=True)\n    df_historical_transactions_copy_group = pd.concat([df_historical_transactions_copy_group, df_historical_transactions_copy_group_dummies], axis=1)\n    df_historical_transactions_copy_group_dummies = pd.get_dummies(df_historical_transactions_copy_group['feature_2'], prefix='feature_2')#, drop_first=True)\n    df_historical_transactions_copy_group = pd.concat([df_historical_transactions_copy_group, df_historical_transactions_copy_group_dummies], axis=1)\n    df_historical_transactions_copy_group_dummies = pd.get_dummies(df_historical_transactions_copy_group['feature_3'], prefix='feature_3', drop_first=True)\n    df_historical_transactions_copy_group = pd.concat([df_historical_transactions_copy_group, df_historical_transactions_copy_group_dummies], axis=1)\n\n    return df_historical_transactions_copy_group\n\ndef pengaik_miracle(df_testing_train):\n    month_lags = range(-13, 1)\n\n    # Group the data by card_id and month_lag\n    grouped = df_testing_train.groupby(['card_id', 'month_lag'])\n\n    # Create a new DataFrame to hold the filled data\n    new_data = []\n\n    # Loop over all the card_id and month_lag combinations we want to fill in\n    for card_id, month_lag in [(c, m) for c in df_testing_train.card_id.unique() for m in month_lags]:\n\n        # If the current (card_id, month_lag) combination is present in the data, use its data\n        if (card_id, month_lag) not in grouped.groups:\n            new_data.append(pd.Series({\n                'card_id': card_id,\n                \n                'month_lag': month_lag,\n                'purchase_amount': -0.7469077955\n            }))\n\n    # Convert the filled data to a DataFrame\n    df_new = pd.DataFrame(new_data)\n\n    #Concat new_df to original\n    df_transactions_filled = pd.concat([df_testing_train, df_new], axis=0)\n\n    # Sort the DataFrame by card_id and month_lag\n    df_transactions_filled.sort_values(['card_id', 'month_lag'], inplace=True)\n\n    # Reset the index of the DataFrame\n    df_transactions_filled.reset_index(drop=True, inplace=True)\n\n    return df_transactions_filled\n\nModelRegressor = Union[LinearRegression, DecisionTreeRegressor, RandomForestRegressor]\ndef feature_selection(approach:str=\"RFE\", \n                      k:int=10, \n                      train:DataFrame=None, \n                      test:DataFrame=None,\n                      model:ModelRegressor=None)->List[str]:\n    \"\"\"This function performs feature selection based on the user's choice of approach.\n    \n    Usage\n    -----\n    >> features = feature_selection(approach=\"LGBM\", train=X_train, test=y_train)\n    \"\"\"\n    # TODO: Sanity check for input\n    # TODO: Documentation for inputs\n    if approach == 'LGBM':\n        # LGTM Regressor to pick out important features\n        gbm = lgb.LGBMRegressor()\n        gbm.fit(train, test)\n\n        # Feature Important Viz\n        fea_imp_ = pd.DataFrame({'variable':train.columns, 'feature_importance':gbm.feature_importances_})\n        return list(fea_imp_[:k]['variable'])\n    if approach == 'RFE':\n        rfe = RFE(estimator=model, n_features_to_select=k)\n        rfe = rfe.fit(train, test)\n\n        # summarize the ranking of the attributes\n        fea_rank_ = pd.DataFrame({'variable':X_train.columns, 'feature_importance':rfe.ranking_})\n        return list(fea_imp_[:k]['variable'])\n\n    \ndef build_train_test_sets(df:DataFrame=None, \n                          features:List[str]=None, \n                          target:str=None, \n                          verbose:int=0, \n                          **kwargs:dict)->List[DataFrame]:\n    \"\"\"This function splits the given dataframe into train and test data.\n    \n    Parameters\n    ----------\n    Args:\n        df: DataFrame\n        This specifies the source DataFrame.\n\n        features: list of str\n        This list containing a str-type elements specifies the name of the features.\n\n        target: str\n        This specifies the target variable.\n        \n        verbose: int-type\n            This species the verbosity of the function.\n    \n    Kwargs\n        A dict mapping the corresponding parameters for scikit learn model selection. \n        \n        {\"test_size\": 0.05, \"seed\": None}\n    \n        If a key from the keys argument is missing from the settings, the default will be used.\n        \n    Returns\n    -------\n    list: This species the list containing X_train, X_test, y_test and y_train DataFrame.\n    \"\"\"\n    # Default \n    model_params = {\n        'test_size': 0.05,\n        \"seed\": None\n    }\n    \n    # Sanity Check\n    if df is None:\n        raise ValueError(\"Expected a DataFrame, no DataFrame supplied.\")\n        \n    if features is None:\n        raise Exception(\"Expected a features list, no features list supplied.\")\n    \n    if not isinstance(features, list):\n        raise Exception(f\"Expected list datatype for features, {type(features)} was supplied.\")\n        \n    if not isinstance(target, str):\n        raise Exception(f\"Expected str datatype for target, {type(target)} was supplied.\")\n        \n    # Check for Kwargs\n    if \"test_size\" in kwargs:\n        model_params['test_size'] = kwargs[\"test_size\"]\n    if \"seed\" in kwargs:\n        model_params['seed'] = kwargs[\"seed\"]\n        \n    seed = \"No Seed\" if model_params[\"seed\"] is None else model_params['seed']\n    if verbose != 0:\n        print(f\"***Parameters for Model Selection***\\ntest_size {model_params['test_size']}\\nseed: {seed}\\n\")\n    \n    if seed != \"No Seed\":\n        X_train, X_test, y_train, y_test = train_test_split(df[features], df[[target]], test_size=model_params['test_size'])\n        return X_train, X_test, y_train, y_test\n    else:\n        X_train, X_test, y_train, y_test = train_test_split(df[features], df[[target]], test_size=model_params['test_size'], random_state=model_params['seed'])\n        return X_train, X_test, y_train, y_test\n    \ndef train_eval_model(model:ModelRegressor,  X_train, X_test, y_train, y_test, name)->DataFrame:\n    \"\"\"This function trains and evaluates the model.\n    By default, the score used it RMSE.\n\n    Parameters\n    ----------\n\n    Args:\n        model: model\n        This species the model to use for training and evaluation.\n       \n    Kwargs\n        A dict mapping the corresponding parameters for training and test data. \n        \n        {\"X_train\": ..., \"X_test\": ..., \"y_train\": ..., \"y_test\": ...}\n    \n    Return \n    ------\n    dataframe: This specifies resets from testing the model.\n    \"\"\"\n    # Train\n    model.fit(X_train, y_train)\n\n    # Pred\n    y_pred = model.predict(X_test)\n\n    # Eval\n    print(f\"{name} Score:\", math.sqrt(mean_squared_error(y_test, y_pred)))\n    \n    return y_pred","metadata":{"execution":{"iopub.status.busy":"2023-03-27T01:12:55.90897Z","iopub.execute_input":"2023-03-27T01:12:55.909411Z","iopub.status.idle":"2023-03-27T01:12:55.964921Z","shell.execute_reply.started":"2023-03-27T01:12:55.909363Z","shell.execute_reply":"2023-03-27T01:12:55.963619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Datasets <a id=\"datasets\"></a>\n\n1. Customer has a ```card_id``` as uuid.\n2. Each customer can make at least one transaction to merchants.\n3. Merchant has ```merchant_id``` as uuid.","metadata":{}},{"cell_type":"code","source":"# Historical Transactions\ntp = pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/historical_transactions.csv', iterator=True, chunksize=2_000_000)  # gives TextFileReader, which is iterable with chunks of 1000 rows.\ndf_historical_transactions = pd.concat(tp, ignore_index=True) \n\n# New Historical Transactions\ntp = pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/new_merchant_transactions.csv', iterator=True, chunksize=2_000_000)  # gives TextFileReader, which is iterable with chunks of 1000 rows.\ndf_new_historical_transactions = pd.concat(tp, ignore_index=True) \n\n# Train Data\ndf_train = pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-27T01:12:55.966206Z","iopub.execute_input":"2023-03-27T01:12:55.967151Z","iopub.status.idle":"2023-03-27T01:14:33.188973Z","shell.execute_reply.started":"2023-03-27T01:12:55.967113Z","shell.execute_reply":"2023-03-27T01:14:33.187709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_new_historical_transactions.columns","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-03-27T01:14:33.19115Z","iopub.execute_input":"2023-03-27T01:14:33.192044Z","iopub.status.idle":"2023-03-27T01:14:33.199426Z","shell.execute_reply.started":"2023-03-27T01:14:33.192011Z","shell.execute_reply":"2023-03-27T01:14:33.197767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rows, cols = df_historical_transactions.shape\nprint(f\"Rows: {format(rows, '_')}\\nCols: {cols}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-27T01:14:33.200606Z","iopub.execute_input":"2023-03-27T01:14:33.200972Z","iopub.status.idle":"2023-03-27T01:14:33.217487Z","shell.execute_reply.started":"2023-03-27T01:14:33.200931Z","shell.execute_reply":"2023-03-27T01:14:33.216027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rows, cols = df_new_historical_transactions.shape\nprint(f\"Rows: {format(rows, '_')}\\nCols: {cols}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-27T01:14:33.218732Z","iopub.execute_input":"2023-03-27T01:14:33.219124Z","iopub.status.idle":"2023-03-27T01:14:33.235907Z","shell.execute_reply.started":"2023-03-27T01:14:33.21909Z","shell.execute_reply":"2023-03-27T01:14:33.234221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rows, cols = df_train.shape\nprint(f\"Rows: {format(rows, '_')}\\nCols: {cols}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-27T01:14:33.237142Z","iopub.execute_input":"2023-03-27T01:14:33.237516Z","iopub.status.idle":"2023-03-27T01:14:33.250565Z","shell.execute_reply.started":"2023-03-27T01:14:33.237485Z","shell.execute_reply":"2023-03-27T01:14:33.248749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge Transactions Data (New and Historical)\ndf_transactions = pd.concat([df_historical_transactions, df_new_historical_transactions])","metadata":{"execution":{"iopub.status.busy":"2023-03-27T01:14:33.251965Z","iopub.execute_input":"2023-03-27T01:14:33.252314Z","iopub.status.idle":"2023-03-27T01:14:39.943291Z","shell.execute_reply.started":"2023-03-27T01:14:33.252285Z","shell.execute_reply":"2023-03-27T01:14:39.939996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_transactions.head(3)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T01:14:39.946729Z","iopub.execute_input":"2023-03-27T01:14:39.947193Z","iopub.status.idle":"2023-03-27T01:14:39.996359Z","shell.execute_reply.started":"2023-03-27T01:14:39.94716Z","shell.execute_reply":"2023-03-27T01:14:39.991094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_transactions.tail(3)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T01:14:40.003249Z","iopub.execute_input":"2023-03-27T01:14:40.003655Z","iopub.status.idle":"2023-03-27T01:14:40.024407Z","shell.execute_reply.started":"2023-03-27T01:14:40.003614Z","shell.execute_reply":"2023-03-27T01:14:40.023239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Probably don't need to use ```merchant_category_id``` and ```merchant_id``` for training as seen from previous version of the notebook (not uploaded).","metadata":{}},{"cell_type":"code","source":"# Remove merchant_category_id and merchant_id\ndf_transactions.drop(columns=['merchant_category_id', 'merchant_id'], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T01:14:40.025808Z","iopub.execute_input":"2023-03-27T01:14:40.026457Z","iopub.status.idle":"2023-03-27T01:14:51.302099Z","shell.execute_reply.started":"2023-03-27T01:14:40.026415Z","shell.execute_reply":"2023-03-27T01:14:51.300603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Imputation <a id='imputation'></a>","metadata":{}},{"cell_type":"code","source":"# Show number of unique groups in the DataFrame\nn_unique = len(df_transactions['card_id'].unique())\nprint(\"Number of Unique CardID Groups:\", format(n_unique, \"_\"))","metadata":{"execution":{"iopub.status.busy":"2023-03-27T01:14:51.30366Z","iopub.execute_input":"2023-03-27T01:14:51.30412Z","iopub.status.idle":"2023-03-27T01:14:53.541148Z","shell.execute_reply.started":"2023-03-27T01:14:51.304085Z","shell.execute_reply":"2023-03-27T01:14:53.540034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Impute with Mode\ndf_transactions = preprocess_data(df=df_transactions)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T01:14:53.54258Z","iopub.execute_input":"2023-03-27T01:14:53.542864Z","iopub.status.idle":"2023-03-27T01:18:13.719118Z","shell.execute_reply.started":"2023-03-27T01:14:53.542826Z","shell.execute_reply":"2023-03-27T01:18:13.717523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Engineering <a id='feature-engineering'></a>","metadata":{}},{"cell_type":"code","source":"# Execute Peng Aik's Miracle\ndf_transactions = pengaik_miracle(df_transactions)","metadata":{"execution":{"iopub.status.busy":"2023-03-27T01:18:13.721227Z","iopub.execute_input":"2023-03-27T01:18:13.721601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Feature Engineering\ndf_transactions = feature_engineering(df_transactions)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge Transactions and Train\ndf_train_merge = merge_data('card_id', [df_train, df_transactions])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Execute Zhen Jie's Miracle\ndf_train_merge = zhenjie_miracle(df_train_merge)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove Missing Data Cause of Test Data\ndf_train_merge.dropna(inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Selection <a id='feature-selection'></a>","metadata":{}},{"cell_type":"code","source":"# Engineered by Zhen Jie so Remove\ndf_train_merge.drop(columns=['first_active_month'], inplace=True)\ndf_train_merge.drop(columns=['hist_purchase_date_max', 'hist_purchase_date_min'], inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Machine Learning <a id='machine-learning'></a>","metadata":{}},{"cell_type":"code","source":"# Split Dataset\nfeatures = list(df_train_merge.drop(columns=['target', 'card_id']).columns)\ntarget = 'target'\nX_train, X_test, y_train, y_test = build_train_test_sets(df_train_merge, \n                                                        features=features,\n                                                        target='target')\n\nprint(\"X_train:\", X_train.shape)\nprint(\"y_train:\", y_train.shape, end='\\n\\n')\n\nprint(\"X_test:\", X_test.shape)\nprint(\"y_test:\", y_test.shape)\n\n# Feature Selection\ntrain = pd.concat([X_train, X_test], axis=0).reset_index(drop=True)\ntest = pd.concat([y_train, y_test], axis=0).reset_index(drop=True)\n\n# Use LGBM for Selection\nfeatures = feature_selection(approach=\"LGBM\", train=train, test=test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize Model\ngbm = lgb.LGBMRegressor()\nlr = LinearRegression()\ndtr = DecisionTreeRegressor(max_depth=5)\ngbm = lgb.LGBMRegressor()\n\n# Use Features from Feature Selection\nX_train, X_test = X_train[features], X_test[features]\n\n# Threading to Speed Up Comparison\nt1 = threading.Thread(target=train_eval_model, args=(gbm, X_train, X_test, y_train, y_test, 'GBM'))\nt2 = threading.Thread(target=train_eval_model, args=(lr, X_train, X_test, y_train, y_test, 'LR'))\nt3 = threading.Thread(target=train_eval_model, args=(dtr, X_train, X_test, y_train, y_test, 'DTR'))\nt4 = threading.Thread(target=train_eval_model, args=(gbm, X_train, X_test, y_train, y_test, 'LGBM'))\n\nt1.start()\nt2.start()\nt3.start()\nt4.start()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Output seems be very dependent on the spliting, perform KFold CV.\n\nLet's take LGBM and perform KFoldCV.","metadata":{}},{"cell_type":"code","source":"# Perform KFold CV for LGBM\ngbm = lgb.LGBMRegressor()\n\nkf_cv = KFold(n_splits=100, shuffle=True)\nscores = cross_val_score(gbm, \n                         train[features], \n                         test, \n                         scoring='neg_root_mean_squared_error', \n                         cv=kf_cv, \n                         n_jobs=-1, # Concurrency\n                         verbose=1\n                        )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = [-score for score in scores]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Min Score:\", min(scores))\nprint(\"Max Score:\", max(scores))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Hmmm... How does the spread look?\nsns.boxplot(scores)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Hyperparameter Tuning","metadata":{}},{"cell_type":"markdown","source":"From the docs...\n\n```\nFor Better Accuracy:\n- Use large max_bin (may be slower)\n- Use small learning_rate with large num_iterations\n- Use large num_leaves (may cause over-fitting)\n- Use bigger training data\n- Try dart\n\nDeal with Over-fitting\n- Use small max_bin\n- Use small num_leaves\n- Use min_data_in_leaf and min_sum_hessian_in_leaf\n- Use bagging by set bagging_fraction and bagging_freq\n- Use feature sub-sampling by set feature_fraction\n- Use bigger training data\n- Try lambda_l1, lambda_l2 and min_gain_to_split for regularization\n- Try max_depth to avoid growing deep tree\n- Try extra_trees\n- Try increasing path_smooth\n```","metadata":{}},{"cell_type":"code","source":"# GridSearchCV\ndtr = DecisionTreeRegressor()\nparameters = {\n    'criterion': ['gini', 'entropy', 'log_loss'],\n    'splitter': ['best', 'random'],\n    'max_depth': [_ for _ in range(10, 110, 10)],\n    'max_leaf_nodes': [_ for _ in range(1, 11, 1)],\n}\ngsearch_dtr GridSearchCV(estimator=dtr, param_grid=parameters, n_jobs=10, cv=10, verbose=1)\ngsearch_dtr.fit(X_train,y_train)\n\n# Get Best Params\ngsearch_dtr.best_params_","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}