{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport datetime\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# ML Tools\nfrom sklearn.model_selection import StratifiedKFold, train_test_split, RandomizedSearchCV\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.impute import SimpleImputer\n\n# ML ALgo\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.linear_model import Lasso\nimport lightgbm as lgb\n\n# Normal Python lib\nimport math\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# CONSTANTS\nINPUT_DIR = '/kaggle/input/elo-merchant-category-recommendation/'\nOUTPUT_DIR = '/kaggle/working/'\nSEED = 123\nTEST_PERC = 0.05\nnp.random.seed(SEED)","metadata":{"execution":{"iopub.status.busy":"2023-02-19T10:10:32.025023Z","iopub.execute_input":"2023-02-19T10:10:32.025422Z","iopub.status.idle":"2023-02-19T10:10:33.754271Z","shell.execute_reply.started":"2023-02-19T10:10:32.025389Z","shell.execute_reply":"2023-02-19T10:10:33.753054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(f'{INPUT_DIR}train.csv')\ndf_test = pd.read_csv(f'{INPUT_DIR}test.csv')\ndf_hist_trans = pd.read_csv(f'{INPUT_DIR}historical_transactions.csv')\ndf_new_merchant_trans = pd.read_csv(f'{INPUT_DIR}new_merchant_transactions.csv')\ndf_merchant = pd.read_csv(f'{INPUT_DIR}merchants.csv')","metadata":{"execution":{"iopub.status.busy":"2023-02-19T10:10:33.756360Z","iopub.execute_input":"2023-02-19T10:10:33.757582Z","iopub.status.idle":"2023-02-19T10:12:22.075647Z","shell.execute_reply.started":"2023-02-19T10:10:33.757544Z","shell.execute_reply":"2023-02-19T10:12:22.074293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Sneakpeek","metadata":{}},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-19T09:11:13.586435Z","iopub.execute_input":"2023-02-19T09:11:13.586911Z","iopub.status.idle":"2023-02-19T09:11:13.618369Z","shell.execute_reply.started":"2023-02-19T09:11:13.586875Z","shell.execute_reply":"2023-02-19T09:11:13.617385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_hist_trans.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-18T08:30:15.636803Z","iopub.execute_input":"2023-02-18T08:30:15.637500Z","iopub.status.idle":"2023-02-18T08:30:15.657549Z","shell.execute_reply.started":"2023-02-18T08:30:15.637460Z","shell.execute_reply":"2023-02-18T08:30:15.656406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_new_merchant_trans.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-18T08:36:01.346490Z","iopub.execute_input":"2023-02-18T08:36:01.346985Z","iopub.status.idle":"2023-02-18T08:36:01.366253Z","shell.execute_reply.started":"2023-02-18T08:36:01.346948Z","shell.execute_reply":"2023-02-18T08:36:01.365077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merchant.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-18T08:44:01.446813Z","iopub.execute_input":"2023-02-18T08:44:01.447684Z","iopub.status.idle":"2023-02-18T08:44:01.479739Z","shell.execute_reply.started":"2023-02-18T08:44:01.447620Z","shell.execute_reply":"2023-02-18T08:44:01.478622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"orig_datasets = {'train': df_train, 'historical_transactions': df_hist_trans, 'new_merchant_transactions': df_new_merchant_trans, 'merchants': df_merchant}\nfor key in orig_datasets.keys():\n    col_na = orig_datasets[key].isna().any()\n    null_col = col_na[col_na == True].index.tolist()\n    if len(null_col) > 0:\n        print(f'{key}: {null_col}')\n    else:\n        print(f'{key}: All clean')","metadata":{"execution":{"iopub.status.busy":"2023-02-19T08:59:19.566141Z","iopub.execute_input":"2023-02-19T08:59:19.566451Z","iopub.status.idle":"2023-02-19T08:59:26.210145Z","shell.execute_reply.started":"2023-02-19T08:59:19.566404Z","shell.execute_reply":"2023-02-19T08:59:26.208778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in ['avg_sales_lag3', 'avg_sales_lag6', 'avg_sales_lag12', 'category_2']:\n    print(df_merchant[col].isna().sum())","metadata":{"execution":{"iopub.status.busy":"2023-02-19T08:59:26.211652Z","iopub.execute_input":"2023-02-19T08:59:26.212041Z","iopub.status.idle":"2023-02-19T08:59:26.225570Z","shell.execute_reply.started":"2023-02-19T08:59:26.212011Z","shell.execute_reply":"2023-02-19T08:59:26.224387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in ['avg_sales_lag3', 'avg_sales_lag6', 'avg_sales_lag12', 'category_2']:\n    display(df_merchant[df_merchant[col].isna()].head())","metadata":{"execution":{"iopub.status.busy":"2023-02-19T08:59:26.228513Z","iopub.execute_input":"2023-02-19T08:59:26.229059Z","iopub.status.idle":"2023-02-19T08:59:26.335205Z","shell.execute_reply.started":"2023-02-19T08:59:26.229022Z","shell.execute_reply":"2023-02-19T08:59:26.334002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Cleaning","metadata":{}},{"cell_type":"markdown","source":"historical_transactions.csv, new_merchant_transactions.csv\n- Drop any rows that have empty column values\n\nmerchants.csv\n- Do not drop yet. Perhaps `sales` attributes are not useful, perhaps `category_2` is not useful\n","metadata":{}},{"cell_type":"code","source":"for df in [df_hist_trans, df_new_merchant_trans]:\n    df = df.dropna(subset=['category_2', 'category_3', 'merchant_id'], inplace=True)\n\nfor df in [df_hist_trans, df_new_merchant_trans]:\n    col_na = df.isna().any()\n    assert len(col_na[col_na == True].index.tolist()) == 0","metadata":{"execution":{"iopub.status.busy":"2023-02-19T10:12:22.078119Z","iopub.execute_input":"2023-02-19T10:12:22.078609Z","iopub.status.idle":"2023-02-19T10:12:42.615636Z","shell.execute_reply.started":"2023-02-19T10:12:22.078565Z","shell.execute_reply":"2023-02-19T10:12:42.613131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_new_columns(name,aggs):\n    return [name + '_' + k + '_' + agg for k in aggs.keys() for agg in aggs[k]]","metadata":{"execution":{"iopub.status.busy":"2023-02-19T10:12:42.617969Z","iopub.execute_input":"2023-02-19T10:12:42.618608Z","iopub.status.idle":"2023-02-19T10:12:42.626464Z","shell.execute_reply.started":"2023-02-19T10:12:42.618565Z","shell.execute_reply":"2023-02-19T10:12:42.625612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Format column datatype\nfor df in [df_hist_trans,df_new_merchant_trans]:\n    df['purchase_date'] = pd.to_datetime(df['purchase_date'])\n    df['year'] = df['purchase_date'].dt.year\n    df['weekofyear'] = df['purchase_date'].dt.weekofyear\n    df['month'] = df['purchase_date'].dt.month\n    df['dayofweek'] = df['purchase_date'].dt.dayofweek\n    df['weekend'] = (df.purchase_date.dt.weekday >=5).astype(int)\n    df['hour'] = df['purchase_date'].dt.hour\n    df['authorized_flag'] = df['authorized_flag'].map({'Y':1, 'N':0})\n    df['category_1'] = df['category_1'].map({'Y':1, 'N':0}) \n    #https://www.kaggle.com/c/elo-merchant-category-recommendation/discussion/73244\n    df['month_diff'] = ((datetime.datetime.today() - df['purchase_date']).dt.days)//30\n    df['month_diff'] += df['month_lag']","metadata":{"execution":{"iopub.status.busy":"2023-02-19T10:12:42.629272Z","iopub.execute_input":"2023-02-19T10:12:42.630343Z","iopub.status.idle":"2023-02-19T10:13:24.132956Z","shell.execute_reply.started":"2023-02-19T10:12:42.630304Z","shell.execute_reply":"2023-02-19T10:13:24.131538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Feature engineering for df_hist_trans DF\naggs = {}\nfor col in ['month','hour','weekofyear','dayofweek','year','subsector_id','merchant_id','merchant_category_id']:\n    aggs[col] = ['nunique']\n\naggs['purchase_amount'] = ['sum','max','min','mean','var']\naggs['installments'] = ['sum','max','min','mean','var']\naggs['purchase_date'] = ['max','min']\naggs['month_lag'] = ['max','min','mean','var']\naggs['month_diff'] = ['mean']\naggs['authorized_flag'] = ['sum', 'mean']\naggs['weekend'] = ['sum', 'mean']\naggs['category_1'] = ['sum', 'mean']\naggs['card_id'] = ['size']\n\nfor col in ['category_2','category_3']:\n    df_hist_trans[col+'_mean'] = df_hist_trans.groupby([col])['purchase_amount'].transform('mean')\n    aggs[col+'_mean'] = ['mean']    \n\nnew_columns = get_new_columns('hist',aggs)\ndf_hist_trans_group = df_hist_trans.groupby('card_id').agg(aggs)\ndf_hist_trans_group.columns = new_columns\ndf_hist_trans_group.reset_index(drop=False,inplace=True)\ndf_hist_trans_group['hist_purchase_date_diff'] = (df_hist_trans_group['hist_purchase_date_max'] - df_hist_trans_group['hist_purchase_date_min']).dt.days\ndf_hist_trans_group['hist_purchase_date_average'] = df_hist_trans_group['hist_purchase_date_diff']/df_hist_trans_group['hist_card_id_size']\ndf_hist_trans_group['hist_purchase_date_uptonow'] = (datetime.datetime.today() - df_hist_trans_group['hist_purchase_date_max']).dt.days\ndf_train = df_train.merge(df_hist_trans_group,on='card_id',how='left')\ndf_test = df_test.merge(df_hist_trans_group,on='card_id',how='left')\n# del df_hist_trans_group","metadata":{"execution":{"iopub.status.busy":"2023-02-19T10:13:24.134580Z","iopub.execute_input":"2023-02-19T10:13:24.134984Z","iopub.status.idle":"2023-02-19T10:14:41.065682Z","shell.execute_reply.started":"2023-02-19T10:13:24.134951Z","shell.execute_reply":"2023-02-19T10:14:41.064537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"orig_datasets = {'hist_trans_group': df_hist_trans_group, 'train': df_train, 'historical_transactions': df_hist_trans, 'new_merchant_transactions': df_new_merchant_trans, 'merchants': df_merchant}\nfor key in orig_datasets.keys():\n    col_na = orig_datasets[key].isna().any()\n    null_col = col_na[col_na == True].index.tolist()\n    if len(null_col) > 0:\n        print(f'{key}: {null_col}')\n    else:\n        print(f'{key}: All clean')","metadata":{"execution":{"iopub.status.busy":"2023-02-19T10:14:41.067517Z","iopub.execute_input":"2023-02-19T10:14:41.068840Z","iopub.status.idle":"2023-02-19T10:14:45.264492Z","shell.execute_reply.started":"2023-02-19T10:14:41.068792Z","shell.execute_reply":"2023-02-19T10:14:45.263233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Feature engineering for df_new_merchant_trans DF\naggs = {}\nfor col in ['month','hour','weekofyear','dayofweek','year','subsector_id','merchant_id','merchant_category_id']:\n    aggs[col] = ['nunique']\naggs['purchase_amount'] = ['sum','max','min','mean','var']\naggs['installments'] = ['sum','max','min','mean','var']\naggs['purchase_date'] = ['max','min']\naggs['month_lag'] = ['max','min','mean','var']\naggs['month_diff'] = ['mean']\naggs['weekend'] = ['sum', 'mean']\naggs['category_1'] = ['sum', 'mean']\naggs['card_id'] = ['size']\n\nfor col in ['category_2','category_3']:\n    df_new_merchant_trans[col+'_mean'] = df_new_merchant_trans.groupby([col])['purchase_amount'].transform('mean')\n    aggs[col+'_mean'] = ['mean']\n    \nnew_columns = get_new_columns('new_hist',aggs)\ndf_hist_trans_group = df_new_merchant_trans.groupby('card_id').agg(aggs)\ndf_hist_trans_group.columns = new_columns\ndf_hist_trans_group.reset_index(drop=False,inplace=True)\ndf_hist_trans_group['new_hist_purchase_date_diff'] = (df_hist_trans_group['new_hist_purchase_date_max'] - df_hist_trans_group['new_hist_purchase_date_min']).dt.days\ndf_hist_trans_group['new_hist_purchase_date_average'] = df_hist_trans_group['new_hist_purchase_date_diff']/df_hist_trans_group['new_hist_card_id_size']\ndf_hist_trans_group['new_hist_purchase_date_uptonow'] = (datetime.datetime.today() - df_hist_trans_group['new_hist_purchase_date_max']).dt.days\ndf_train = df_train.merge(df_hist_trans_group,on='card_id',how='left')\ndf_test = df_test.merge(df_hist_trans_group,on='card_id',how='left')\ndel df_hist_trans_group","metadata":{"execution":{"iopub.status.busy":"2023-02-19T10:14:45.266769Z","iopub.execute_input":"2023-02-19T10:14:45.267687Z","iopub.status.idle":"2023-02-19T10:14:53.464801Z","shell.execute_reply.started":"2023-02-19T10:14:45.267607Z","shell.execute_reply":"2023-02-19T10:14:53.463766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove outliers\nthres = 3\nzscore = np.abs((df_train['target'] - df_train['target'].mean())/df_train['target'].std())\n\ndf_train['outliers'] = zscore > thres\ndf_train['outliers'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-02-19T10:14:53.466171Z","iopub.execute_input":"2023-02-19T10:14:53.466696Z","iopub.status.idle":"2023-02-19T10:14:53.484826Z","shell.execute_reply.started":"2023-02-19T10:14:53.466648Z","shell.execute_reply":"2023-02-19T10:14:53.483112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for df in [df_train,df_test]:\n    df['first_active_month'] = pd.to_datetime(df['first_active_month'])\n    df['dayofweek'] = df['first_active_month'].dt.dayofweek\n    df['weekofyear'] = df['first_active_month'].dt.weekofyear\n    df['month'] = df['first_active_month'].dt.month\n    df['elapsed_time'] = (datetime.datetime.today() - df['first_active_month']).dt.days\n    df['hist_first_buy'] = (df['hist_purchase_date_min'] - df['first_active_month']).dt.days\n    df['new_hist_first_buy'] = (df['new_hist_purchase_date_min'] - df['first_active_month']).dt.days\n    for f in ['hist_purchase_date_max','hist_purchase_date_min','new_hist_purchase_date_max',\\\n                     'new_hist_purchase_date_min']:\n        df[f] = df[f].astype(np.int64) * 1e-9\n    df['card_id_total'] = df['new_hist_card_id_size']+df['hist_card_id_size']\n    df['purchase_amount_total'] = df['new_hist_purchase_amount_sum']+df['hist_purchase_amount_sum']\n\n# Hypothesise that the features are nominal, hence normalise (we should experiment the absence of this case)\nfor f in ['feature_1','feature_2','feature_3']:\n    order_label = df_train.groupby([f])['outliers'].mean()\n    print(order_label)\n    df_train[f] = df_train[f].map(order_label)\n    df_test[f] = df_test[f].map(order_label)","metadata":{"execution":{"iopub.status.busy":"2023-02-19T10:14:53.486738Z","iopub.execute_input":"2023-02-19T10:14:53.487201Z","iopub.status.idle":"2023-02-19T10:14:54.109596Z","shell.execute_reply.started":"2023-02-19T10:14:53.487158Z","shell.execute_reply":"2023-02-19T10:14:54.108111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.groupby(['feature_1'])['outliers'].mean()","metadata":{"execution":{"iopub.status.busy":"2023-02-19T10:14:54.113804Z","iopub.execute_input":"2023-02-19T10:14:54.114378Z","iopub.status.idle":"2023-02-19T10:14:54.129098Z","shell.execute_reply.started":"2023-02-19T10:14:54.114343Z","shell.execute_reply":"2023-02-19T10:14:54.127881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Linear Regression\nExperiment Encoding (for Documentation Purposes)\n- 0.x: ways to tackle NA values\n  - **all** outliers involved\n  - Linear Regression\n  - **all** features are used\n- 1.x: test whether the 3 `feature` variables in `train.csv` are useful\n  - **all** outliers involved\n  - Linear Regression\n  - NA values are filled with Imputer\n- 2.x: test with different regression models\n  - **all** outliers involved\n  - NA values are filled with Imputer\n  - **all** features are used","metadata":{}},{"cell_type":"code","source":"# Experiment 0.1: training data is free from NA. But this greatly reduces the training data\ndf_train_clean = df_train.dropna()\nprint('Percentage reduction: {:%}'.format(1-df_train_clean.size/df_train.size))\n\ndf_train_columns = [c for c in df_train.columns if c not in ['card_id', 'first_active_month','target','outliers']]\ntarget = df_train_clean['target']\ndel df_train_clean['target']\n\nlr = LinearRegression()\n\nX_train, X_test, y_train, y_test = train_test_split(df_train_clean.loc[:,df_train_columns], target, test_size=TEST_PERC, random_state=SEED)\n\nlr.fit(X_train, y_train)\ny_predict = lr.predict(X_test)\nprint(\"Score:\", math.sqrt(mean_squared_error(y_test, y_predict)))","metadata":{"execution":{"iopub.status.busy":"2023-02-19T04:27:00.521602Z","iopub.execute_input":"2023-02-19T04:27:00.522004Z","iopub.status.idle":"2023-02-19T04:27:01.238698Z","shell.execute_reply.started":"2023-02-19T04:27:00.521973Z","shell.execute_reply":"2023-02-19T04:27:01.237622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Experiment 0.2: training data is uses fillna\ndf_train_clean2 = df_train.copy()\nfor col in df_train_clean2.columns:\n    if df_train_clean2[col].isna().sum() > 0:\n        df_train_clean2[col] = df_train_clean2[col].fillna(df_train_clean2[col].mean())\n\ntarget = df_train_clean2['target']\ndel df_train_clean2['target']\n\nlr = LinearRegression()\n\nX_train, X_test, y_train, y_test = train_test_split(df_train_clean2.loc[:,df_train_columns], target, test_size=TEST_PERC, random_state=SEED)\n\nlr.fit(X_train, y_train)\ny_predict = lr.predict(X_test)\nprint(\"Score:\", math.sqrt(mean_squared_error(y_test, y_predict)))","metadata":{"execution":{"iopub.status.busy":"2023-02-19T04:27:02.951197Z","iopub.execute_input":"2023-02-19T04:27:02.951553Z","iopub.status.idle":"2023-02-19T04:27:03.692653Z","shell.execute_reply.started":"2023-02-19T04:27:02.951527Z","shell.execute_reply":"2023-02-19T04:27:03.691755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Experiment 1.x: use all data except feature_1\ndef experiment(feature_to_remove):\n    df_train_columns = [c for c in df_train.columns if c not in ['card_id', 'first_active_month','target','outliers'] + feature_to_remove]\n    target = df_train['target']\n    \n    # Clean nan values\n    imputer = SimpleImputer(missing_values=np.nan, strategy='mean')\n    X = df_train.loc[:,df_train_columns]\n    X = imputer.fit_transform(X)\n    \n    # Split data\n    X_train, X_test, y_train, y_test = train_test_split(X, target, test_size=TEST_PERC, random_state=SEED)\n    \n    lr = LinearRegression()\n    lr.fit(X_train, y_train)\n    y_predict = lr.predict(X_test)\n    print(\"Score after removing\", feature_to_remove, \":\", math.sqrt(mean_squared_error(y_test, y_predict)))\n\nexperiment([])  # 1.1\nexperiment(['feature_1'])  # 1.2\nexperiment(['feature_2'])  # 1.3\nexperiment(['feature_3'])  # 1.4\nexperiment(['feature_1', 'feature_2'])  # 1.5\nexperiment(['feature_2', 'feature_3'])  # 1.6\nexperiment(['feature_1', 'feature_3'])  # 1.7\nexperiment(['feature_1', 'feature_2', 'feature_3'])  # 1.8","metadata":{"execution":{"iopub.status.busy":"2023-02-19T09:17:01.424105Z","iopub.execute_input":"2023-02-19T09:17:01.424547Z","iopub.status.idle":"2023-02-19T09:17:09.319598Z","shell.execute_reply.started":"2023-02-19T09:17:01.424509Z","shell.execute_reply":"2023-02-19T09:17:09.318690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Experiment 2.x: test with different regression models\ndef experiment_2x(model):\n    df_train_columns = [c for c in df_train.columns if c not in ['card_id', 'first_active_month','target','outliers']]\n    target = df_train['target']\n    \n    # Clean nan values\n    imputer = SimpleImputer(missing_values=np.nan, strategy='mean')\n    X = df_train.loc[:,df_train_columns]\n    X = imputer.fit_transform(X)\n    \n    # Split data\n    X_train, X_test, y_train, y_test = train_test_split(X, target, test_size=TEST_PERC, random_state=SEED)\n\n    model.fit(X_train, y_train)\n    y_predict = model.predict(X_test)\n    print(\"Score for model\", type(model).__name__, \":\", math.sqrt(mean_squared_error(y_test, y_predict)))\n    \nexperiment_2x(LinearRegression())  # 2.1\nexperiment_2x(Lasso(alpha=0.1))    # 2.2","metadata":{"execution":{"iopub.status.busy":"2023-02-19T10:29:31.817730Z","iopub.execute_input":"2023-02-19T10:29:31.818247Z","iopub.status.idle":"2023-02-19T10:29:50.856434Z","shell.execute_reply.started":"2023-02-19T10:29:31.818208Z","shell.execute_reply":"2023-02-19T10:29:50.855180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Experiment 2.2.x: Hyperparameter tuning of Lasso\ndef experiment_2_2x():\n    ### Data Processing ###\n    df_train_columns = [c for c in df_train.columns if c not in ['card_id', 'first_active_month','target','outliers']]\n    target = df_train['target']\n    \n    # Clean nan values\n    imputer = SimpleImputer(missing_values=np.nan, strategy='mean')\n    X = df_train.loc[:,df_train_columns]\n    X = imputer.fit_transform(X)\n    \n    # Split data\n    X_train, X_test, y_train, y_test = train_test_split(X, target, test_size=TEST_PERC, random_state=SEED)    \n\n    ### Tuning Code ###\n    model = Lasso()\n    hp = {\n        'alpha': np.linspace(0.01, 10, 100)\n    }\n    random_search = RandomizedSearchCV(estimator=model, param_distributions=hp, n_iter=5, cv=5, random_state=SEED)\n    random_search.fit(X_train, y_train)\n    print(\"Best hyperparams:\", random_search.best_params_)\n    return random_search.best_estimator_\n\nmodel_best_lasso = experiment_2_2x()","metadata":{"execution":{"iopub.status.busy":"2023-02-19T11:16:32.203274Z","iopub.execute_input":"2023-02-19T11:16:32.204051Z","iopub.status.idle":"2023-02-19T11:21:55.177198Z","shell.execute_reply.started":"2023-02-19T11:16:32.204007Z","shell.execute_reply":"2023-02-19T11:21:55.174125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import PCA\n\ndef pca_vis():\n    ### Data Processing ###\n    df_train_columns = [c for c in df_train.columns if c not in ['card_id', 'first_active_month','target','outliers']]\n    target = df_train['target']\n\n    # Clean nan values\n    imputer = SimpleImputer(missing_values=np.nan, strategy='mean')\n    X = df_train.loc[:,df_train_columns]\n    X = imputer.fit_transform(X)\n\n    # Standardisation\n    scaler = StandardScaler()\n    scaler.fit(X)\n    scaled_data = scaler.transform(X)\n    \n    # PCA\n    pca = PCA(n_components=2)\n    pca.fit(scaled_data)\n    x_pca = pca.transform(scaled_data)\n    \n    # Plot\n    plt.figure(figsize=(8,6))\n    plt.scatter(x_pca[:,0], x_pca[:,1], c=target, cmap='plasma')\n    plt.xlabel('First component')\n    plt.ylabel('Second component')\n    \n#     df_pca = pd.DataFrame(pca.components_, columns=target)\n#     plt.figure(figsize=(12,6))\n#     sns.heatmap(df_pca, cmap='plasma')\n    \n    return pca\n\npca = pca_vis()","metadata":{"execution":{"iopub.status.busy":"2023-02-19T11:25:09.059496Z","iopub.execute_input":"2023-02-19T11:25:09.059960Z","iopub.status.idle":"2023-02-19T11:25:20.228462Z","shell.execute_reply.started":"2023-02-19T11:25:09.059920Z","shell.execute_reply":"2023-02-19T11:25:20.227136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate File for Kaggle Submission","metadata":{}},{"cell_type":"code","source":"# In the combined df_test, some card_ids simply have NA values. Test with fillna with the column's mean\ndf_test_clean = df_test.copy()\nfor col in df_test_clean.columns:\n    if df_test_clean[col].isna().sum() > 0:\n        df_test_clean[col] = df_test_clean[col].fillna(df_test_clean[col].mean())\n\nprint('Percentage reduction: {:%}'.format(1-df_test_clean.size/df_test.size))\ny_test_predict = lr.predict(df_test_clean.loc[:,df_train_columns])","metadata":{"execution":{"iopub.status.busy":"2023-02-19T04:27:15.973082Z","iopub.execute_input":"2023-02-19T04:27:15.973472Z","iopub.status.idle":"2023-02-19T04:27:16.142337Z","shell.execute_reply.started":"2023-02-19T04:27:15.973441Z","shell.execute_reply":"2023-02-19T04:27:16.141362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_predict = pd.DataFrame({'card_id': df_test_clean.card_id.unique(), 'target': y_test_predict})\ndf_test_predict.to_csv('d1.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-02-19T04:27:16.178698Z","iopub.execute_input":"2023-02-19T04:27:16.179449Z","iopub.status.idle":"2023-02-19T04:27:16.508712Z","shell.execute_reply.started":"2023-02-19T04:27:16.179412Z","shell.execute_reply":"2023-02-19T04:27:16.507593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Observations\n1. Some `card_id` does not have values in `hist_trans` or `new_hist_trans`, resulting in empty rows. This disrupts the generation of Kaggle test predictions, as some `card_id` simply does not have these values. Unless we drop these values?\n - Or do we split into two models: one for `card_id` with complete details, one for that without?\n \n2. Experiments of removing `train.csv` `feature`s in all 7 combinations show that the `feature`s have no impact on prediction, as score is always about 3.83","metadata":{}},{"cell_type":"markdown","source":"# Next Action\n1. Two models: one for outlier, one for normal data\n2. Reapply experiments 1.x and 2.x on the top 30 features\n3. PCA() to identify the top 30 features","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}