{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":10445,"databundleVersionId":200747,"sourceType":"competition"},{"sourceId":8817736,"sourceType":"datasetVersion","datasetId":5304571},{"sourceId":8833115,"sourceType":"datasetVersion","datasetId":5305385}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-07-01T13:54:46.921953Z","iopub.execute_input":"2024-07-01T13:54:46.922352Z","iopub.status.idle":"2024-07-01T13:54:48.170145Z","shell.execute_reply.started":"2024-07-01T13:54:46.922318Z","shell.execute_reply":"2024-07-01T13:54:48.168818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def miss_handle(se):\n    if(se.isna().sum()==0):\n        return se\n    if(se.dtypes==\"object\"):\n        vc = se.value_counts()\n        if(vc.iloc[0]/len(se)>0.1):\n            se = se.fillna(-1)\n        else:\n            se = se.fillna(se.mode().iloc[0])\n    else:\n        se = se.fillna(se.mean())\n    return se\n\ndef inf_handle(se):\n    if((se==np.inf).any()):\n        se_ = se.replace(np.inf,np.nan)\n        se = se_.fillna(se_.max())\n    if((se==-np.inf).any()):\n        se_ = se.replace(-np.inf,np.nan)\n        se = se_.fillna(se_.min())\n    return se\n\ndef pre_handle(df):\n    inf_col = df.columns[(df==np.inf).sum()>0]\n    inf_col = inf_col.union(df.columns[(df==-np.inf).sum()>0])\n    df[inf_col] = df[inf_col].apply(inf_handle,axis=0)\n    miss_col = df.columns[df.isna().sum()>0]\n    df[miss_col] = df[miss_col].apply(miss_handle,axis=0)\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-07-01T14:31:54.811399Z","iopub.execute_input":"2024-07-01T14:31:54.811854Z","iopub.status.idle":"2024-07-01T14:31:54.827197Z","shell.execute_reply.started":"2024-07-01T14:31:54.811818Z","shell.execute_reply":"2024-07-01T14:31:54.825907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions = pd.read_csv(\"/kaggle/input/elo-merchant-category-recommendation/transactions.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-07-01T14:31:57.183461Z","iopub.execute_input":"2024-07-01T14:31:57.183849Z","iopub.status.idle":"2024-07-01T14:35:25.104950Z","shell.execute_reply.started":"2024-07-01T14:31:57.183822Z","shell.execute_reply":"2024-07-01T14:35:25.103301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_col = pd.Index([\"merchant_id\",'city_id', 'merchant_category_id'])\nnum_col = pd.Index([\"merchant_id\",'city_id', 'merchant_category_id','purchase_amount', 'purchase_date',\n       'index', 'merchant_group_id', 'numerical_1', 'numerical_2',\n       'avg_sales_lag3', 'avg_purchases_lag3', 'avg_sales_lag6',\n       'avg_purchases_lag6', 'avg_sales_lag12', 'avg_purchases_lag12'])\nobj_col = pd.Index(['active_months_lag12', 'active_months_lag3', 'active_months_lag6',\n       'authorized_flag', 'category_1', 'category_2', 'category_3',\n       'category_4', 'installments', 'month_lag',\n       'most_recent_purchases_range', 'most_recent_sales_range', 'state_id',\n       'subsector_id'])\n\nnum_col_dict = {}\ntp = [\"median\",\"mean\",\"std\",\"max\",\"min\",\"skew\"]\nfor t in tp:\n    num_col_dict[t] = []\n    for col in num_col:\n        num_col_dict[t].append(f\"{col}_{t}\")\n    num_col_dict[t] = pd.Index(num_col_dict[t])","metadata":{"execution":{"iopub.status.busy":"2024-07-01T15:17:08.531179Z","iopub.execute_input":"2024-07-01T15:17:08.531879Z","iopub.status.idle":"2024-07-01T15:17:08.542500Z","shell.execute_reply.started":"2024-07-01T15:17:08.531834Z","shell.execute_reply":"2024-07-01T15:17:08.541383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions[\"merchant_id\"] = transactions[\"merchant_id\"].map(lambda x: eval(f\"0x{x.split('_')[-1]}\"))\ntransactions[num_col] = transactions[num_col].apply(lambda x:(x-x.mean())/x.std(),axis=0)\n\ngrouped_trans = transactions.groupby('card_id')\n\nnew_data = pd.DataFrame()\nfor t in tp:\n    t_col = num_col_dict[t]\n    new_data[t_col] = eval(f\"grouped_trans[num_col].{t}()\")\nnew_data[\"count\"] = grouped_trans[num_col[0]].count()\nfor col in id_col:\n    new_data[f\"{col}_mode\"] = grouped_trans[col].apply(lambda x: x.mode().iloc[0])","metadata":{"execution":{"iopub.status.busy":"2024-07-01T15:29:40.835254Z","iopub.execute_input":"2024-07-01T15:29:40.835741Z","iopub.status.idle":"2024-07-01T15:29:42.481284Z","shell.execute_reply.started":"2024-07-01T15:29:40.835706Z","shell.execute_reply":"2024-07-01T15:29:42.479697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.decomposition import PCA \n\n# pca = PCA(n_components=2) \n# pac_f = lambda x:pd.Series(pca.fit_transform(x.T).flatten())\n# pca_results = grouped_trans[num_col].apply(pac_f)\n# pca_results.columns = [f\"pca{i+1}\" for i in range(len(num_col)*2)] \n# pca_results[\"card_id\"] = pca_results.index\n# pca_results.to_csv(\"/kaggle/working/pca_results.csv\",index=False)\n\npca_results = pd.read_csv(\"/kaggle/input/elo-merchant-category-recommendation-train-test/pca_results.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-07-01T15:25:01.883087Z","iopub.execute_input":"2024-07-01T15:25:01.883448Z","iopub.status.idle":"2024-07-01T15:25:06.306804Z","shell.execute_reply.started":"2024-07-01T15:25:01.883409Z","shell.execute_reply":"2024-07-01T15:25:06.305798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"obj_data = pd.DataFrame()\nfor col in obj_col:\n    obj_dum = pd.get_dummies(transactions[['card_id',col]],columns=[col]).groupby('card_id').sum()\n    obj_data = pd.concat((obj_data,obj_dum),axis=1)\n    obj_data[f\"{col}_nunique\"] = transactions[['card_id',col]].groupby('card_id').nunique()\nobj_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-01T15:25:57.295181Z","iopub.execute_input":"2024-07-01T15:25:57.295579Z","iopub.status.idle":"2024-07-01T15:26:09.935658Z","shell.execute_reply.started":"2024-07-01T15:25:57.295550Z","shell.execute_reply":"2024-07-01T15:26:09.934531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_data = pd.concat((obj_data,new_data),axis=1)\nnew_data = pd.merge(new_data,pca_results,on=[\"card_id\"],how=\"left\")","metadata":{"execution":{"iopub.status.busy":"2024-07-01T15:26:30.120609Z","iopub.execute_input":"2024-07-01T15:26:30.121718Z","iopub.status.idle":"2024-07-01T15:26:30.741944Z","shell.execute_reply.started":"2024-07-01T15:26:30.121673Z","shell.execute_reply":"2024-07-01T15:26:30.740575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(\"/kaggle/input/c/elo-merchant-category-recommendation/test.csv\")\ntrain = pd.read_csv(\"/kaggle/input/c/elo-merchant-category-recommendation/train.csv\")\n\ntest = pre_handle(test)\ntrain = pre_handle(train)\n\ntest[\"first_active_month\"] = pd.to_datetime(test[\"first_active_month\"]).apply(lambda x:x.timestamp()-1.5e9)\ntest[\"first_active_month\"] = (test[\"first_active_month\"]-test[\"first_active_month\"].mean())/test[\"first_active_month\"].std()\ntrain[\"first_active_month\"] = pd.to_datetime(train[\"first_active_month\"]).apply(lambda x:x.timestamp()-1.5e9)\ntrain[\"first_active_month\"] = (train[\"first_active_month\"]-train[\"first_active_month\"].mean())/train[\"first_active_month\"].std()\n\ntrain_tag = train[[\"card_id\",\"target\"]]\ntest_tag = test[[\"card_id\"]]\npre_train_data = pd.concat((train[train.columns.drop(\"target\")],test),axis=0)\n\npre_train_data = pd.get_dummies(pre_train_data,columns=[\"feature_1\",\"feature_2\",\"feature_3\"])","metadata":{"execution":{"iopub.status.busy":"2024-07-01T15:26:32.044285Z","iopub.execute_input":"2024-07-01T15:26:32.044780Z","iopub.status.idle":"2024-07-01T15:26:34.567415Z","shell.execute_reply.started":"2024-07-01T15:26:32.044746Z","shell.execute_reply":"2024-07-01T15:26:34.566216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_data = pd.merge(new_data,pre_train_data,on=\"card_id\",how=\"right\")\ntrain_data = pd.merge(new_data,train_tag,on=\"card_id\",how=\"right\")\ntest_data = pd.merge(new_data,test_tag,on=\"card_id\",how=\"right\")","metadata":{"execution":{"iopub.status.busy":"2024-07-01T15:26:37.069271Z","iopub.execute_input":"2024-07-01T15:26:37.069723Z","iopub.status.idle":"2024-07-01T15:26:39.066049Z","shell.execute_reply.started":"2024-07-01T15:26:37.069688Z","shell.execute_reply":"2024-07-01T15:26:39.064872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in train_data.columns:\n    print(i)","metadata":{"execution":{"iopub.status.busy":"2024-07-01T15:28:02.276522Z","iopub.execute_input":"2024-07-01T15:28:02.276937Z","iopub.status.idle":"2024-07-01T15:28:02.283146Z","shell.execute_reply.started":"2024-07-01T15:28:02.276907Z","shell.execute_reply":"2024-07-01T15:28:02.282042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.to_csv(\"/kaggle/working/train_data.csv\",index=False)\ntest_data.to_csv(\"/kaggle/working/test_data.csv\",index=False)","metadata":{},"execution_count":null,"outputs":[]}]}