{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":10445,"databundleVersionId":200747,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-14T10:52:31.639685Z","iopub.execute_input":"2024-05-14T10:52:31.640350Z","iopub.status.idle":"2024-05-14T10:52:32.829704Z","shell.execute_reply.started":"2024-05-14T10:52:31.640306Z","shell.execute_reply":"2024-05-14T10:52:32.828788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import the necessary libraries\nimport numpy as np\nimport pandas as pd\nimport os\nimport time\nimport warnings\nimport gc\ngc.collect()\nimport os\nfrom six.moves import urllib\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport datetime\nwarnings.filterwarnings('ignore')\n%matplotlib inline\nplt.style.use('seaborn')\nfrom scipy.stats import norm, skew\nfrom sklearn.preprocessing import StandardScaler","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:52:52.649599Z","iopub.execute_input":"2024-05-14T10:52:52.649959Z","iopub.status.idle":"2024-05-14T10:52:53.409939Z","shell.execute_reply.started":"2024-05-14T10:52:52.649929Z","shell.execute_reply":"2024-05-14T10:52:53.408558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Add All the Models Libraries\n\n# Scalers\nfrom sklearn.utils import shuffle\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.pipeline import FeatureUnion\n\n# Models\n\nfrom sklearn.linear_model import Lasso\nfrom sklearn.metrics import mean_squared_log_error,mean_squared_error, r2_score,mean_absolute_error\n\nfrom sklearn.model_selection import train_test_split #training and testing data split\nfrom sklearn import metrics #accuracy measure\nfrom sklearn.metrics import confusion_matrix #for confusion matrix\nfrom scipy.stats import reciprocal, uniform\n\nfrom sklearn.model_selection import StratifiedKFold, RepeatedKFold\n\n# Cross-validation\nfrom sklearn.model_selection import KFold #for K-fold cross validation\nfrom sklearn.model_selection import cross_val_score #score evaluation\nfrom sklearn.model_selection import cross_val_predict #prediction\nfrom sklearn.model_selection import cross_validate\n\n# GridSearchCV\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import RandomizedSearchCV\n\n#Common data processors\nfrom sklearn.preprocessing import OneHotEncoder, LabelEncoder\nfrom sklearn import feature_selection\nfrom sklearn import model_selection\nfrom sklearn import metrics\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.utils import check_array\nfrom scipy import sparse","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:53:03.469743Z","iopub.execute_input":"2024-05-14T10:53:03.470106Z","iopub.status.idle":"2024-05-14T10:53:03.879063Z","shell.execute_reply.started":"2024-05-14T10:53:03.470077Z","shell.execute_reply":"2024-05-14T10:53:03.877884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# to make this notebook's output stable across runs\nnp.random.seed(123)\ngc.collect()\n# To plot pretty figures\n%matplotlib inline\nplt.rcParams['axes.labelsize'] = 14\nplt.rcParams['xtick.labelsize'] = 12\nplt.rcParams['ytick.labelsize'] = 12","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:53:12.335061Z","iopub.execute_input":"2024-05-14T10:53:12.335445Z","iopub.status.idle":"2024-05-14T10:53:12.423313Z","shell.execute_reply.started":"2024-05-14T10:53:12.335411Z","shell.execute_reply":"2024-05-14T10:53:12.421841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Reduce the memory usage - Inspired by Panchajanya Banerjee\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:53:28.356827Z","iopub.execute_input":"2024-05-14T10:53:28.357207Z","iopub.status.idle":"2024-05-14T10:53:28.370179Z","shell.execute_reply.started":"2024-05-14T10:53:28.357179Z","shell.execute_reply":"2024-05-14T10:53:28.368973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def one_hot_encoder(df, nan_as_category = True):\n    original_columns = list(df.columns)\n    categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n    df = pd.get_dummies(df, columns= categorical_columns, dummy_na= nan_as_category)\n    new_columns = [c for c in df.columns if c not in original_columns]\n    return df, new_columns","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:53:39.563471Z","iopub.execute_input":"2024-05-14T10:53:39.563834Z","iopub.status.idle":"2024-05-14T10:53:39.570315Z","shell.execute_reply.started":"2024-05-14T10:53:39.563807Z","shell.execute_reply":"2024-05-14T10:53:39.568870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = reduce_mem_usage(pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/train.csv',parse_dates=[\"first_active_month\"]))\ntest = reduce_mem_usage(pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/train.csv', parse_dates=[\"first_active_month\"]))","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:20.810409Z","iopub.execute_input":"2024-05-14T10:54:20.810776Z","iopub.status.idle":"2024-05-14T10:54:21.517242Z","shell.execute_reply.started":"2024-05-14T10:54:20.810749Z","shell.execute_reply":"2024-05-14T10:54:21.516152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:41.459383Z","iopub.execute_input":"2024-05-14T10:54:41.459747Z","iopub.status.idle":"2024-05-14T10:54:41.468673Z","shell.execute_reply.started":"2024-05-14T10:54:41.459721Z","shell.execute_reply":"2024-05-14T10:54:41.467332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.subplot(1, 2, 2)\nsns.distplot(train.target, kde=True, fit = norm)\nplt.xlabel('Customer Loyality (Skewed)')","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:54:57.732435Z","iopub.execute_input":"2024-05-14T10:54:57.732835Z","iopub.status.idle":"2024-05-14T10:54:59.075084Z","shell.execute_reply.started":"2024-05-14T10:54:57.732805Z","shell.execute_reply":"2024-05-14T10:54:59.073816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove the Outliers \ntrain['outliers'] = 0\ntrain.loc[train['target'] < -30, 'outliers'] = 1\ntrain['outliers'].value_counts()\n\nfor features in ['feature_1','feature_2','feature_3']:\n    order_label = train.groupby([features])['outliers'].mean()\n    train[features] = train[features].map(order_label)\n    test[features] =  test[features].map(order_label)","metadata":{"execution":{"iopub.status.busy":"2024-05-14T10:55:19.711417Z","iopub.execute_input":"2024-05-14T10:55:19.711789Z","iopub.status.idle":"2024-05-14T10:55:19.757369Z","shell.execute_reply.started":"2024-05-14T10:55:19.711761Z","shell.execute_reply":"2024-05-14T10:55:19.756306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Feature Engineering\ntrain[\"month\"] = train[\"first_active_month\"].dt.month\ntrain[\"year\"] = train[\"first_active_month\"].dt.year\ntrain['week'] = train[\"first_active_month\"].dt.year\ntrain['dayofweek'] = train['first_active_month'].dt.dayofweek\ntrain['days'] = (pd.Timestamp(2018,2, 1) - train['first_active_month']).dt.days\n\ntrain['days_feature1_ratio'] = train['feature_1'] / train['days']\ntrain['days_feature2_ratio'] = train['feature_2'] / train['days']\ntrain['days_feature3_ratio'] = train['feature_3'] / train['days']\n\ntrain['feature_sum'] = train['feature_1'] + train['feature_2'] + train['feature_3']\ntrain['feature_mean'] = train['feature_sum']/3\ntrain['feature_max'] = train[['feature_1', 'feature_2', 'feature_3']].max(axis=1)\ntrain['feature_min'] = train[['feature_1', 'feature_2', 'feature_3']].min(axis=1)\ntrain['feature_var'] = train[['feature_1', 'feature_2', 'feature_3']].std(axis=1)\n\ntest[\"month\"] = test[\"first_active_month\"].dt.month\ntest[\"year\"] = test[\"first_active_month\"].dt.year\ntest['week'] = test[\"first_active_month\"].dt.year\ntest['dayofweek'] = test['first_active_month'].dt.dayofweek\ntest['days'] = (pd.Timestamp(2018,2, 1) - test['first_active_month']).dt.days\n\ntest['days_feature1_ratio'] = test['feature_1'] / train['days']\ntest['days_feature2_ratio'] = test['feature_2'] / train['days']\ntest['days_feature3_ratio'] = test['feature_3'] / train['days']\n\ntest['feature_sum'] = test['feature_1'] + test['feature_2'] + test['feature_3']\ntest['feature_mean'] = test['feature_sum']/3\ntest['feature_max'] = test[['feature_1', 'feature_2', 'feature_3']].max(axis=1)\ntest['feature_min'] = test[['feature_1', 'feature_2', 'feature_3']].min(axis=1)\ntest['feature_var'] = test[['feature_1', 'feature_2', 'feature_3']].std(axis=1)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T11:01:21.212895Z","iopub.execute_input":"2024-05-14T11:01:21.213303Z","iopub.status.idle":"2024-05-14T11:01:21.972731Z","shell.execute_reply.started":"2024-05-14T11:01:21.213269Z","shell.execute_reply":"2024-05-14T11:01:21.971335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def aggregate_transaction_hist(trans, prefix):  \n        \n    agg_func = {\n        'purchase_amount' : ['sum','max','min','mean'],\n        'installments' : ['sum','max','mean'],\n        'card_id' : ['count'],\n        'price' :['sum','mean'],\n    }\n    #对购买金额进行聚合统计，包括总和、最大值、最小值、均值\n    # 对分期付款数量进行聚合统计，包括总和、最大值、均值,这些特征可以用来描述用户的分期付款行为\n    #card_id: 对卡号进行聚合统计，包括交易数量和交易次数。这些特征可以用来描述用户的交易活跃度。\n    #统计用户的购买金额的总和和均值情况\n    \n    agg_trans = trans.groupby(['card_id']).agg(agg_func)\n    agg_trans.columns = [prefix + '_'.join(col).strip() \n                           for col in agg_trans.columns.values]\n    agg_trans.reset_index(inplace=True)\n    \n    df = (trans.groupby('card_id')\n          .size()\n          .reset_index(name='{}transactions_count'.format(prefix)))\n    \n    agg_trans = pd.merge(df, agg_trans, on='card_id', how='left')\n    \n    return agg_trans","metadata":{"execution":{"iopub.status.busy":"2024-05-14T11:07:24.333875Z","iopub.execute_input":"2024-05-14T11:07:24.334279Z","iopub.status.idle":"2024-05-14T11:07:24.342060Z","shell.execute_reply.started":"2024-05-14T11:07:24.334234Z","shell.execute_reply":"2024-05-14T11:07:24.340782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions = reduce_mem_usage(pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/historical_transactions.csv'))\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T11:08:03.709538Z","iopub.execute_input":"2024-05-14T11:08:03.710309Z","iopub.status.idle":"2024-05-14T11:09:59.921971Z","shell.execute_reply.started":"2024-05-14T11:08:03.710267Z","shell.execute_reply":"2024-05-14T11:09:59.919814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions['category_2'] = transactions['category_2'].fillna(1.0,inplace=True)\ntransactions['category_3'] = transactions['category_3'].fillna('A',inplace=True)\ntransactions['merchant_id'] = transactions['merchant_id'].fillna('M_ID_00a6ca8a8a',inplace=True)\ntransactions['installments'].replace(-1, np.nan,inplace=True)\ntransactions['installments'].replace(999, np.nan,inplace=True)\ntransactions['purchase_amount'] = transactions['purchase_amount'].apply(lambda x: min(x, 0.8))\n\n#Feature Engineering\ntransactions['authorized_flag'] = transactions['authorized_flag'].map({'Y': 1, 'N': 0})\ntransactions['category_1'] = transactions['category_1'].map({'Y': 1, 'N': 0})\ntransactions['category_3'] = transactions['category_3'].map({'A':0, 'B':1, 'C':2})\n\ntransactions['purchase_date'] = pd.to_datetime(transactions['purchase_date'])\ntransactions['year'] = transactions['purchase_date'].dt.year\ntransactions['weekofyear'] = transactions['purchase_date'].dt.isocalendar().week\ntransactions['month'] = transactions['purchase_date'].dt.month\ntransactions['hour'] = transactions['purchase_date'].dt.hour \ntransactions['month_diff'] = ((datetime.datetime.today() - transactions['purchase_date']).dt.days)//30\ntransactions['month_diff'] += transactions['month_lag']\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T11:18:14.973452Z","iopub.execute_input":"2024-05-14T11:18:14.973897Z","iopub.status.idle":"2024-05-14T11:18:58.383469Z","shell.execute_reply.started":"2024-05-14T11:18:14.973861Z","shell.execute_reply":"2024-05-14T11:18:58.382273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nagg_func = {\n        'mean': ['mean'],\n    }\nfor col in ['category_2','category_3']:\n    transactions[col+'_sum'] = transactions['purchase_amount'].groupby(transactions[col]).agg('sum')\n    transactions[col+'_mean'] = transactions['purchase_amount'].groupby(transactions[col]).agg('mean')\n    transactions[col+'_max'] = transactions['purchase_amount'].groupby(transactions[col]).agg('max')\n    transactions[col+'_min'] = transactions['purchase_amount'].groupby(transactions[col]).agg('min')\n    agg_func[col+'_mean'] = ['mean']\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T11:20:04.775947Z","iopub.execute_input":"2024-05-14T11:20:04.776376Z","iopub.status.idle":"2024-05-14T11:20:31.551521Z","shell.execute_reply.started":"2024-05-14T11:20:04.776345Z","shell.execute_reply":"2024-05-14T11:20:31.550332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transactions['price'] = transactions['purchase_amount'] / transactions['installments']\n","metadata":{"execution":{"iopub.status.busy":"2024-05-14T11:25:19.831736Z","iopub.execute_input":"2024-05-14T11:25:19.832126Z","iopub.status.idle":"2024-05-14T11:25:20.016642Z","shell.execute_reply.started":"2024-05-14T11:25:19.832097Z","shell.execute_reply":"2024-05-14T11:25:20.015405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nmerge_trans = aggregate_transaction_hist(transactions, prefix='hist_')\ndel transactions\ngc.collect()\ntrain = pd.merge(train, merge_trans, on='card_id',how='left')\ntest = pd.merge(test, merge_trans, on='card_id',how='left')\ndel merge_trans\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T11:25:22.096450Z","iopub.execute_input":"2024-05-14T11:25:22.096816Z","iopub.status.idle":"2024-05-14T11:25:35.541931Z","shell.execute_reply.started":"2024-05-14T11:25:22.096790Z","shell.execute_reply":"2024-05-14T11:25:35.540737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Now extract the data from the new transactions\nnew_transactions = reduce_mem_usage(pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/new_merchant_transactions.csv'))","metadata":{"execution":{"iopub.status.busy":"2024-05-14T11:33:01.424206Z","iopub.execute_input":"2024-05-14T11:33:01.424682Z","iopub.status.idle":"2024-05-14T11:33:08.579111Z","shell.execute_reply.started":"2024-05-14T11:33:01.424647Z","shell.execute_reply":"2024-05-14T11:33:08.577561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_transactions['category_2'] = new_transactions['category_2'].fillna(1.0,inplace=True)\nnew_transactions['category_3'] = new_transactions['category_3'].fillna('A',inplace=True)\nnew_transactions['merchant_id'] = new_transactions['merchant_id'].fillna('M_ID_00a6ca8a8a',inplace=True)\nnew_transactions['installments'].replace(-1, np.nan,inplace=True)\nnew_transactions['installments'].replace(999, np.nan,inplace=True)\nnew_transactions['purchase_amount'] = new_transactions['purchase_amount'].apply(lambda x: min(x, 0.8))\n\n#Feature Engineering\nnew_transactions['authorized_flag'] = new_transactions['authorized_flag'].map({'Y': 1, 'N': 0})\nnew_transactions['category_1'] = new_transactions['category_1'].map({'Y': 1, 'N': 0})\nnew_transactions['category_3'] = new_transactions['category_3'].map({'A':0, 'B':1, 'C':2})\n\nnew_transactions['purchase_date'] = pd.to_datetime(new_transactions['purchase_date'])\nnew_transactions['year'] = new_transactions['purchase_date'].dt.year\nnew_transactions['weekofyear'] = new_transactions['purchase_date'].dt.isocalendar().week\nnew_transactions['month'] = new_transactions['purchase_date'].dt.month\nnew_transactions['hour'] = new_transactions['purchase_date'].dt.hour \nnew_transactions['month_diff'] = ((datetime.datetime.today() - new_transactions['purchase_date']).dt.days)//30\nnew_transactions['month_diff'] += new_transactions['month_lag']\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T11:36:14.896539Z","iopub.execute_input":"2024-05-14T11:36:14.897009Z","iopub.status.idle":"2024-05-14T11:36:20.801807Z","shell.execute_reply.started":"2024-05-14T11:36:14.896973Z","shell.execute_reply":"2024-05-14T11:36:20.800583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg_func = {\n        'mean': ['mean'],\n    }\nfor col in ['category_2','category_3']:\n    new_transactions[col+'_sum'] = new_transactions['purchase_amount'].groupby(new_transactions[col]).agg('sum')\n    new_transactions[col+'_mean'] = new_transactions['purchase_amount'].groupby(new_transactions[col]).agg('mean')\n    new_transactions[col+'_max'] = new_transactions['purchase_amount'].groupby(new_transactions[col]).agg('max')\n    new_transactions[col+'_min'] = new_transactions['purchase_amount'].groupby(new_transactions[col]).agg('min')\n    agg_func[col+'_mean'] = ['mean']\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T11:38:22.432706Z","iopub.execute_input":"2024-05-14T11:38:22.433082Z","iopub.status.idle":"2024-05-14T11:38:23.515447Z","shell.execute_reply.started":"2024-05-14T11:38:22.433055Z","shell.execute_reply":"2024-05-14T11:38:23.514214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_transactions['price'] = new_transactions['purchase_amount'] / new_transactions['installments']\n","metadata":{"execution":{"iopub.status.busy":"2024-05-14T11:38:53.402960Z","iopub.execute_input":"2024-05-14T11:38:53.403386Z","iopub.status.idle":"2024-05-14T11:38:53.415460Z","shell.execute_reply.started":"2024-05-14T11:38:53.403353Z","shell.execute_reply":"2024-05-14T11:38:53.414224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def aggregate_transaction_new(trans, prefix):  \n    agg_func = {\n        'purchase_amount' : ['sum','max','min','mean'],\n        'installments' : ['sum','max','mean'],\n        'card_id' : ['count'],\n        'price' :['sum','mean'],\n    }\n    agg_trans = trans.groupby(['card_id']).agg(agg_func)\n    agg_trans.columns = [prefix + '_'.join(col).strip() \n                           for col in agg_trans.columns.values]\n    agg_trans.reset_index(inplace=True)\n    \n    df = (trans.groupby('card_id')\n          .size()\n          .reset_index(name='{}transactions_count'.format(prefix)))\n    \n    agg_trans = pd.merge(df, agg_trans, on='card_id', how='left')\n    \n    return agg_trans","metadata":{"execution":{"iopub.status.busy":"2024-05-14T11:42:24.371425Z","iopub.execute_input":"2024-05-14T11:42:24.371866Z","iopub.status.idle":"2024-05-14T11:42:24.379987Z","shell.execute_reply.started":"2024-05-14T11:42:24.371831Z","shell.execute_reply":"2024-05-14T11:42:24.378500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merge_new = aggregate_transaction_new(new_transactions, prefix='new_')\ndel new_transactions\ngc.collect()\n\ntrain = pd.merge(train, merge_new, on='card_id',how='left')\ntest = pd.merge(test, merge_new, on='card_id',how='left')\ndel merge_new\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-14T11:42:28.535109Z","iopub.execute_input":"2024-05-14T11:42:28.536061Z","iopub.status.idle":"2024-05-14T11:42:32.847370Z","shell.execute_reply.started":"2024-05-14T11:42:28.536021Z","shell.execute_reply":"2024-05-14T11:42:32.846133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['card_id_cnt_total'] =  train['new_card_id_count']+train['hist_card_id_count']","metadata":{"execution":{"iopub.status.busy":"2024-05-14T11:42:47.777398Z","iopub.execute_input":"2024-05-14T11:42:47.777785Z","iopub.status.idle":"2024-05-14T11:42:47.783803Z","shell.execute_reply.started":"2024-05-14T11:42:47.777757Z","shell.execute_reply":"2024-05-14T11:42:47.782959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['purchase_amount_total'] =  train['new_purchase_amount_sum']+train['hist_purchase_amount_sum']","metadata":{"execution":{"iopub.status.busy":"2024-05-14T11:43:51.167334Z","iopub.execute_input":"2024-05-14T11:43:51.170685Z","iopub.status.idle":"2024-05-14T11:43:51.199329Z","shell.execute_reply.started":"2024-05-14T11:43:51.170524Z","shell.execute_reply":"2024-05-14T11:43:51.196650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['card_id_cnt_total'] =  test['new_card_id_count']+test['hist_card_id_count']\ntest['purchase_amount_total'] = test['new_purchase_amount_sum']+test['hist_purchase_amount_sum']","metadata":{"execution":{"iopub.status.busy":"2024-05-14T11:45:07.985152Z","iopub.execute_input":"2024-05-14T11:45:07.985538Z","iopub.status.idle":"2024-05-14T11:45:07.995779Z","shell.execute_reply.started":"2024-05-14T11:45:07.985511Z","shell.execute_reply":"2024-05-14T11:45:07.994247Z"},"trusted":true},"execution_count":null,"outputs":[]}]}