{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":10445,"databundleVersionId":200747,"sourceType":"competition"}],"dockerImageVersionId":30761,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm,tqdm_notebook \nimport lightgbm as lgb\nimport xgboost as xgb\nfrom catboost import CatBoostRegressor\nfrom catboost import CatBoostClassifier\nfrom sklearn.linear_model import Ridge, RidgeCV\nfrom sklearn.linear_model import BayesianRidge\nfrom catboost import CatBoostRegressor\nfrom sklearn.model_selection import KFold, StratifiedKFold, RepeatedKFold\nfrom scipy import sparse\nimport warnings\nimport time\nimport sys\nimport os\nimport gc\nimport datetime\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import log_loss \nwarnings.simplefilter(action='ignore', category=FutureWarning) \nwarnings.filterwarnings(\"ignore\") \npd.set_option('display.max_columns',None) \npd.set_option('max_colwidth',100)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-18T19:50:46.145674Z","iopub.execute_input":"2024-09-18T19:50:46.146131Z","iopub.status.idle":"2024-09-18T19:50:50.588505Z","shell.execute_reply.started":"2024-09-18T19:50:46.146081Z","shell.execute_reply":"2024-09-18T19:50:50.587268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Memory management\n\n# %%time\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64'] \n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtypes \n        if col_type in numerics: \n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64) \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float32).max: \n                    df[col] = df[col].astype(np.float32)\n                #elif c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max: \n                    #df[col] = df[col].astype(np.float16)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1%} reduction)'.format(end_mem, (start_mem-end_mem)/start_mem))\n    return df\n\nnew_transactions = pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/new_merchant_transactions.csv', parse_dates=['purchase_date']) \nhistorical_transactions = pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/historical_transactions.csv', parse_dates=['purchase_date'])\nfor col in ['authorized_flag', 'category_1']:\n    historical_transactions[col] = historical_transactions[col].map({'Y':1, 'N':0}) \n    new_transactions[col] = new_transactions[col].map({'Y':1, 'N':0})","metadata":{"execution":{"iopub.status.busy":"2024-09-17T23:13:33.698367Z","iopub.execute_input":"2024-09-17T23:13:33.698835Z","iopub.status.idle":"2024-09-17T23:15:32.078220Z","shell.execute_reply.started":"2024-09-17T23:13:33.698800Z","shell.execute_reply":"2024-09-17T23:15:32.076225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Read training/testing dataset and preprosess.\n\ntrain = pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/train.csv') \ntest = pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/test.csv')\n\ntarget = train['target'] \nfor df in [train, test]:\n    df['year'] = df['first_active_month'].fillna('0-0').apply(lambda x:int(str(x).split('-')[0])) \n    df['first_active_month'] = pd.to_datetime(df['first_active_month'])\n    df['elapsed_time'] = pd.to_timedelta(datetime.date(2018,3,1) - df['first_active_month'].dt.date).dt.days\n    \n    df['first_active_month'] = pd.to_datetime(df['first_active_month']) \n    df['weekofyear'] = df['first_active_month'].dt.isocalendar().week\n    df['dayofyear'] = df['first_active_month'].dt.dayofyear\n    df['month'] = df['first_active_month'].dt.month\n    \n## Merge transactions with train and test\ntrain_test = pd.concat([train[['card_id','first_active_month']], test[['card_id','first_active_month']]], axis=0, ignore_index=True)\nhistorical_transactions = historical_transactions.merge(train_test[['card_id','first_active_month']], on=['card_id'], how='left') \nnew_transactions = new_transactions.merge(train_test[['card_id','first_active_month']], on=['card_id'], how='left')","metadata":{"execution":{"iopub.status.busy":"2024-09-17T23:15:32.080420Z","iopub.execute_input":"2024-09-17T23:15:32.080792Z","iopub.status.idle":"2024-09-17T23:15:45.185696Z","shell.execute_reply.started":"2024-09-17T23:15:32.080749Z","shell.execute_reply":"2024-09-17T23:15:45.184625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## \ndef month_trans(x): \n    return x // 30\ndef week_trans(x): \n    return x // 7\n\n## Preprocess of transaction data\ndef get_expand_common(df_): \n    df = df_.copy()\n    df['category_2'].fillna(1.0,inplace=True) \n    df['category_3'].fillna('A',inplace=True)\n    df['category_3'] = df['category_3'].map({'A':0, 'B':1, 'C':2}) \n    df['merchant_id'].fillna('M_ID_00a6ca8a8a',inplace=True) \n    df['installments'].replace(-1, np.nan,inplace=True) \n    df['installments'].replace(999, np.nan,inplace=True) \n    df['installments'].replace(0, 1,inplace=True)\n    df['purchase_amount'] = np.round(df['purchase_amount'] / 0.00150265118 + 497.06,8) \n    df['purchase_amount'] = df.purchase_amount.apply(lambda x: np.round(x))\n\n    df['purchase_date'] = pd.to_datetime(df['purchase_date'])\n    df['first_active_month'] = pd.to_datetime(df['first_active_month'])\n    df['purchase_hour'] = df['purchase_date'].dt.hour\n    df['year'] = df['purchase_date'].dt.year\n    df['month'] = df['purchase_date'].dt.month\n    df['day'] = df['purchase_date'].dt.day\n    df['hour'] = df['purchase_date'].dt.hour\n    df['weekofyear'] = df['purchase_date'].dt.isocalendar().week\n    df['dayofweek'] = df['purchase_date'].dt.dayofweek\n    df['weekend'] = (df.purchase_date.dt.weekday>=5).astype(int)\n    df = df.sort_values(['card_id','purchase_date'])\n    df['purchase_date_floorday'] = df['purchase_date'].dt.floor('d')\n\n    # time since active day, 0,1,2,3,...,max-act\n    df['purchase_day_since_active_day'] = df['purchase_date_floorday'] - df['first_active_month']  \n    df['purchase_day_since_active_day']  = df['purchase_day_since_active_day'].dt.days   #.astype('timedelta64[D]')\n    df['purchase_month_since_active_day'] = df['purchase_day_since_active_day'].agg(month_trans).values \n    df['purchase_week_since_active_day'] = df['purchase_day_since_active_day'].agg(week_trans).values\n    \n    # time since reference day to the last purchase, 0,1,2,3,...,max-act\n    ht_card_id_gp = df.groupby('card_id')\n    df['purchase_day_since_reference_day'] = ht_card_id_gp['purchase_date_floorday'].transform('max') - df['purchase_date_floorday']\n    df['purchase_day_since_reference_day'] = df['purchase_day_since_reference_day'].dt.days\n    df['purchase_week_since_reference_day'] = df['purchase_day_since_reference_day'].agg(week_trans).values\n    df['purchase_month_since_reference_day'] = df['purchase_day_since_reference_day'].agg(month_trans).values \n    \n    # time interval between two purchases\n    df['purchase_day_diff'] = df['purchase_date_floorday'].shift()\n    df['purchase_day_diff'] = df['purchase_date_floorday'].values - df['purchase_day_diff'].values\n    df['purchase_day_diff'] = df['purchase_day_diff'].dt.days\n    df['purchase_week_diff'] = df['purchase_day_diff'].agg(week_trans).values \n    df['purchase_month_diff'] = df['purchase_day_diff'].agg(month_trans).values\n    \n    df['purchase_amount_ddgd_98']  = df['purchase_amount'].values * df['purchase_day_since_reference_day'].apply(lambda x:0.98**x).values \n    df['purchase_amount_ddgd_99']  = df['purchase_amount'].values * df['purchase_day_since_reference_day'].apply(lambda x:0.99**x).values     \n    df['purchase_amount_wdgd_96']  = df['purchase_amount'].values * df['purchase_week_since_reference_day'].apply(lambda x:0.96**x).values  \n    df['purchase_amount_wdgd_97']  = df['purchase_amount'].values * df['purchase_week_since_reference_day'].apply(lambda x:0.97**x).values  \n    df['purchase_amount_mdgd_90']  = df['purchase_amount'].values * df['purchase_month_since_reference_day'].apply(lambda x:0.9**x).values \n    df['purchase_amount_mdgd_80']  = df['purchase_amount'].values * df['purchase_month_since_reference_day'].apply(lambda x:0.8**x).values  \n\n    df = reduce_mem_usage(df)\n    return df\n  \nhistorical_transactions = get_expand_common(historical_transactions) \nnew_transactions = get_expand_common(new_transactions)                                                                   ","metadata":{"execution":{"iopub.status.busy":"2024-09-17T23:15:45.187492Z","iopub.execute_input":"2024-09-17T23:15:45.187927Z","iopub.status.idle":"2024-09-17T23:21:22.637720Z","shell.execute_reply.started":"2024-09-17T23:15:45.187882Z","shell.execute_reply":"2024-09-17T23:21:22.636700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Feature Engingeering\n\n## Generate statistics features\n\ndef aggregate_transactions(df_, prefix): \n    df = df_.copy()\n    \n    df['month_diff'] = ((datetime.datetime.today() - df['purchase_date']).dt.days)//30 \n    df['month_diff'] = df['month_diff'].astype(int)\n    df['month_diff'] += df['month_lag']\n    \n    df['price'] = df['purchase_amount'] / df['installments']\n    df['duration'] = df['purchase_amount'] * df['month_diff'] \n    df['amount_month_ratio'] = df['purchase_amount'] / df['month_diff']\n\n    df.loc[:, 'purchase_date'] = pd.DatetimeIndex(df['purchase_date']).astype(np.int64) * 1e-9\n\n    agg_func = {\n        'category_1':      ['mean'], \n        'category_2':      ['mean'], \n        'category_3':      ['mean'], \n        'installments':    ['mean', 'max', 'min', 'std'], \n        'month_lag':       ['nunique', 'mean', 'max', 'min', 'std'], \n        'month':           ['nunique', 'mean', 'max', 'min', 'std'], \n        'hour':            ['nunique', 'mean', 'max', 'min', 'std'], \n        'weekofyear':      ['nunique', 'mean', 'max', 'min', 'std'], \n        'dayofweek':       ['nunique', 'mean'], \n        'weekend':         ['mean'], \n        'year':            ['nunique'], \n        'card_id':         ['size','count'], \n        'purchase_date':   ['max', 'min'], \n        'price':           ['mean','max','min','std'], \n        'duration':        ['mean','min','max','std','skew'], \n        'amount_month_ratio':['mean','min','max','std','skew'], \n    }  \n    \n    for col in ['category_2','category_3']: \n        df[col+'_mean'] = df.groupby([col])['purchase_amount'].transform('mean') \n        agg_func[col+'_mean'] = ['mean'] \n          \n    agg_df = df.groupby(['card_id']).agg(agg_func) \n    agg_df.columns = [prefix + '_'.join(col).strip() for col in agg_df.columns.values] \n    agg_df.reset_index(drop=False, inplace=True) \n        \n    return agg_df \n\nprint('generate statistics features...') \nauth_base_stat = aggregate_transactions(historical_transactions[historical_transactions['authorized_flag']==1], prefix='auth_') \nprint('generate statistics features...') \nhist_base_stat = aggregate_transactions(historical_transactions[historical_transactions['authorized_flag']==0], prefix='hist_') \nprint('generate statistics features...') \nnew_base_stat = aggregate_transactions(new_transactions, prefix='new_') ","metadata":{"execution":{"iopub.status.busy":"2024-09-17T23:21:22.640301Z","iopub.execute_input":"2024-09-17T23:21:22.640622Z","iopub.status.idle":"2024-09-17T23:22:15.986530Z","shell.execute_reply.started":"2024-09-17T23:21:22.640588Z","shell.execute_reply":"2024-09-17T23:22:15.985630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_quantile(x, percentiles = [0.1, 0.25, 0.75, 0.9]):\n    x_len = len(x)\n    x = np.sort(x)\n    sts_feas = []\n    for per_ in percentiles:\n        if per_ == 1: \n            sts_feas.append(x[x_len - 1])\n        else:\n            sts_feas.append(x[int(x_len * per_)])\n    return sts_feas\n\ndef get_cardf_tran(df_, month = 3, prefix = '_'):\n    df = df_.copy()  \n    if prefix == 'hist_cardf_': \n        df['month_to_now'] = pd.to_timedelta(datetime.date(2018, month, 1) - df['purchase_date_floorday'].dt.date).dt.days \n        \n    df['month_diff'] = (pd.to_timedelta(datetime.datetime.today() - df['purchase_date']).dt.days)//30 \n    df['month_diff'] = df['month_diff'].astype(int) \n    df['month_diff'] += df['month_lag'] \n          \n    print('*' * 30,'Part1, whole data','*' * 30) \n    cardid_features = pd.DataFrame() \n    cardid_features['card_id'] = df['card_id'].unique()    \n    print( '*' * 30, 'Traditional Features', '*' * 30) \n    ht_card_id_gp = df.groupby('card_id')  \n    cardid_features['card_id_cnt'] = ht_card_id_gp['authorized_flag'].count().values \n    \n    if  prefix == 'hist_cardf_': \n        cardid_features['card_id_isau_mean'] = ht_card_id_gp['authorized_flag'].mean().values \n        cardid_features['card_id_isau_sum'] = ht_card_id_gp['authorized_flag'].sum().values  \n          \n    cardid_features['month_diff_mean'] = ht_card_id_gp['month_diff'].mean().values \n    cardid_features['month_diff_median'] = ht_card_id_gp['month_diff'].median().values \n    \n    if prefix == 'hist_cardf_': \n        cardid_features['reference_day'] = ht_card_id_gp['purchase_date_floorday'].max().values \n        cardid_features['first_day'] = ht_card_id_gp['purchase_date_floorday'].min().values  \n        cardid_features['activation_day'] = ht_card_id_gp['first_active_month'].max().values \n             \n        # first to activation day \n        cardid_features['first_to_activation_day'] = pd.to_timedelta(cardid_features['first_day'] - cardid_features['activation_day']).dt.days \n        # activation to reference day  \n        cardid_features['activation_to_reference_day'] = pd.to_timedelta(cardid_features['reference_day'] - cardid_features['activation_day']).dt.days \n        # first to last day  \n        cardid_features['first_to_reference_day'] = pd.to_timedelta(cardid_features['reference_day'] - cardid_features['first_day']).dt.days \n        # reference day to now   \n        cardid_features['reference_day_to_now'] = pd.to_timedelta(datetime.date(2018, month, 1) - cardid_features['reference_day'].dt.date).dt.days  \n        # first day to now \n        cardid_features['first_day_to_now'] = pd.to_timedelta(datetime.date(2018, month, 1) - cardid_features['first_day'].dt.date).dt.days  \n              \n        print('card_id(month_lag, min to reference day):min') \n        cardid_features['card_id_month_lag_min'] = ht_card_id_gp['month_lag'].agg('min').values    \n        # is_purchase_before_activation,first_to_reference_day_divide_activation_to_reference_day \n        cardid_features['is_purchase_before_activation'] = cardid_features['first_to_activation_day'] < 0  \n        cardid_features['is_purchase_before_activation'] = cardid_features['is_purchase_before_activation'].astype(int) \n        cardid_features['first_to_reference_day_divide_activation_to_reference_day'] = cardid_features['first_to_reference_day'] / (cardid_features['activation_to_reference_day'] + 0.01) \n        cardid_features['days_per_count'] = cardid_features['first_to_reference_day'].values / cardid_features['card_id_cnt'].values \n        \n    if prefix == 'new_cardf_': \n        print(' Eight time features, ')  \n        cardid_features['reference_day'] = ht_card_id_gp['reference_day'].last().values \n        cardid_features['first_day'] = ht_card_id_gp['purchase_date_floorday'].min().values  \n        cardid_features['last_day'] = ht_card_id_gp['purchase_date_floorday'].max().values \n        cardid_features['activation_day'] = ht_card_id_gp['first_active_month'].max().values \n        # reference to first day \n        cardid_features['reference_day_to_first_day'] = pd.to_timedelta(cardid_features['first_day'] - cardid_features['reference_day']).dt.days \n        # reference to last day \n        cardid_features['reference_day_to_last_day'] = pd.to_timedelta(cardid_features['last_day'] - cardid_features['reference_day']).dt.days   \n        # first to last day  \n        cardid_features['first_to_last_day'] = pd.to_timedelta(cardid_features['last_day'] - cardid_features['first_day']).dt.days \n        # activation to first day  \n        cardid_features['activation_to_first_day'] = pd.to_timedelta(cardid_features['first_day'] - cardid_features['activation_day']).dt.days \n        # activation to first day  \n        cardid_features['activation_to_last_day'] = pd.to_timedelta(cardid_features['last_day'] - cardid_features['activation_day']).dt.days \n        # last day to now   \n        cardid_features['reference_day_to_now'] = pd.to_timedelta(datetime.date(2018, month, 1) - cardid_features['reference_day'].dt.date).dt.days  \n        # first day to now \n        cardid_features['first_day_to_now'] = pd.to_timedelta(datetime.date(2018, month, 1) - cardid_features['first_day'].dt.date).dt.days  \n    \n        print('card_id(month_lag, min to reference day):min') \n        cardid_features['card_id_month_lag_max'] = ht_card_id_gp['month_lag'].agg('max').values   \n        cardid_features['first_to_last_day_divide_reference_to_last_day'] = cardid_features['first_to_last_day'] / (cardid_features['reference_day_to_last_day'] + 0.01) \n        cardid_features['days_per_count'] = cardid_features['first_to_last_day'].values / cardid_features['card_id_cnt'].values \n        \n    for f in ['reference_day', 'first_day', 'last_day', 'activation_day']: \n        try: \n            del cardid_features[f] \n        except: \n            print(f, 'not exists！') \n\n    print('card id(city_id,installments,merchant_category_id,.......): nunique, cnt/nunique')  \n    for col in tqdm_notebook(['category_1','category_2','category_3','state_id','city_id','installments','merchant_id', 'merchant_category_id','subsector_id','month_lag','purchase_date_floorday']): \n        cardid_features['card_id_%s_nunique'%col] = ht_card_id_gp[col].nunique().values \n        cardid_features['card_id_cnt_divide_%s_nunique'%col] = cardid_features['card_id_cnt'].values / cardid_features['card_id_%s_nunique'%col].values \n\n    print('card_id(purchase_amount & degrade version ): mean,sum,std,median,quantile(10,25,75,90)')  \n    for col in tqdm_notebook(['installments','purchase_amount','purchase_amount_ddgd_98','purchase_amount_ddgd_99','purchase_amount_wdgd_96','purchase_amount_wdgd_97','purchase_amount_mdgd_90','purchase_amount_mdgd_80']): \n        if col == 'purchase_amount': \n            for opt in ['sum','mean','std','median','max','min']: \n                cardid_features['card_id_' +col+ '_' + opt] = ht_card_id_gp[col].agg(opt).values \n                \n            cardid_features['card_id_' +col+ '_range'] = cardid_features['card_id_' +col+ '_max'].values - cardid_features['card_id_' +col+ '_min'].values \n            percentiles = ht_card_id_gp[col].apply(lambda x:get_quantile(x,percentiles = [0.025, 0.25, 0.75, 0.975]))  \n                \n            cardid_features[col + '_2.5_quantile'] = percentiles.map(lambda x:x[0]).values \n            cardid_features[col + '_25_quantile'] = percentiles.map(lambda x:x[1]).values \n            cardid_features[col + '_75_quantile'] = percentiles.map(lambda x:x[2]).values \n            cardid_features[col + '_97.5_quantile'] = percentiles.map(lambda x:x[3]).values \n            cardid_features['card_id_' +col+ '_range2'] = cardid_features[col+ '_97.5_quantile'].values - cardid_features[col+ '_2.5_quantile'].values \n            del cardid_features[col + '_2.5_quantile'],cardid_features[col + '_97.5_quantile'] \n            gc.collect() \n        else: \n            for opt in ['sum']: \n                cardid_features['card_id_' +col+ '_' + opt] = ht_card_id_gp[col].agg(opt).values\n    \n    print( '*' * 30, 'Pivot Features', '*' * 30) \n    print('Count  Pivot') # ignore purchase_month_since_reference_day since it may be duplicated with month_lag; dayofweek, merchant_cate, state_id have little effect \n    \n    for pivot_col in tqdm_notebook(['category_1','category_2','category_3','month_lag','subsector_id','weekend']): #'city_id',\n        tmp = df.groupby(['card_id',pivot_col])['merchant_id'].count().to_frame(pivot_col + '_count') \n        tmp.reset_index(inplace =True)         \n        tmp_pivot = pd.pivot_table(data=tmp,index = 'card_id',columns=pivot_col,values=pivot_col + '_count',fill_value=0) \n        tmp_pivot.columns = [tmp_pivot.columns.names[0] + '_cnt_pivot_'+ str(col) for col in tmp_pivot.columns] \n        tmp_pivot.reset_index(inplace = True) \n        cardid_features = cardid_features.merge(tmp_pivot, on = 'card_id', how='left') \n            \n        if  pivot_col != 'weekend' and pivot_col != 'installments': \n            tmp = df.groupby(['card_id',pivot_col])['purchase_date_floorday'].nunique().to_frame(pivot_col + '_purchase_date_floorday_nunique')  \n            tmp1 = df.groupby(['card_id'])['purchase_date_floorday'].nunique().to_frame('purchase_date_floorday_nunique')  \n            tmp.reset_index(inplace =True)   \n            tmp1.reset_index(inplace =True)    \n            tmp = tmp.merge(tmp1, on ='card_id', how='left') \n            tmp[pivot_col + '_day_nunique_pct'] = tmp[pivot_col + '_purchase_date_floorday_nunique'].values / tmp['purchase_date_floorday_nunique'].values \n            tmp_pivot = pd.pivot_table(data=tmp,index = 'card_id',columns=pivot_col,values=pivot_col + '_day_nunique_pct',fill_value=0) \n            tmp_pivot.columns = [tmp_pivot.columns.names[0] + '_day_nunique_pct_'+ str(col) for col in tmp_pivot.columns] \n            tmp_pivot.reset_index(inplace = True) \n            cardid_features = cardid_features.merge(tmp_pivot, on = 'card_id', how='left')\n   \n    ######## transaction before activate  ##############\n    if prefix == 'new_cardf_':     \n        print('*'*30,'Part2， data with time less than activation day','*'*30) \n        df_part = df.loc[df.purchase_date < df.first_active_month] \n      \n        cardid_features_part = pd.DataFrame() \n        cardid_features_part['card_id'] = df_part['card_id'].unique()    \n        ht_card_id_part_gp = df_part.groupby('card_id') \n        cardid_features_part['card_id_part_cnt'] = ht_card_id_part_gp['authorized_flag'].count().values \n      \n        print('card_id(purchase_amount): sum')  \n        for col in tqdm_notebook(['purchase_amount']):  \n            for opt in ['sum','mean']: \n                cardid_features_part['card_id_part_' +col+ '_' + opt] = ht_card_id_part_gp[col].agg(opt).values \n      \n        cardid_features = cardid_features.merge(cardid_features_part, on ='card_id', how='left') \n        cardid_features['card_id_part_purchase_amount_sum_percent'] = cardid_features['card_id_part_purchase_amount_sum'] / (cardid_features['card_id_purchase_amount_sum'] + 0.01) \n    \n    cardid_features = reduce_mem_usage(cardid_features)\n    \n    new_col_names = [] \n    for col in cardid_features.columns: \n        if col == 'card_id': \n            new_col_names.append(col) \n        else: \n            new_col_names.append(prefix + col) \n    cardid_features.columns = new_col_names \n          \n    return cardid_features \n\nprint('auth...') \nauthorized_transactions = historical_transactions.loc[historical_transactions['authorized_flag'] == 1] \nauth_cardf_tran = get_cardf_tran(authorized_transactions, 3, prefix='auth_cardf_') \nprint('hist...') \nhist_cardf_tran = get_cardf_tran(historical_transactions, 3, prefix='hist_cardf_') \nprint('new...') \nreference_days = historical_transactions.groupby('card_id')['purchase_date'].last().to_frame('reference_day') \nreference_days.reset_index(inplace = True) \nnew_transactions = new_transactions.merge(reference_days, on ='card_id', how='left') \nnew_cardf_tran  = get_cardf_tran(new_transactions, 5, prefix='new_cardf_') ","metadata":{"execution":{"iopub.status.busy":"2024-09-17T23:22:15.988238Z","iopub.execute_input":"2024-09-17T23:22:15.988640Z","iopub.status.idle":"2024-09-17T23:30:51.399213Z","shell.execute_reply.started":"2024-09-17T23:22:15.988596Z","shell.execute_reply":"2024-09-17T23:30:51.398060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Same thing for transactions in last 2 months\ndef get_cardf_tran_last2(df_, month = 3, prefix = 'last2_'):\n    df = df_.loc[df_.month_lag >= -2].copy() \n    print('*'*30,'Part1, whole data','*'*30) \n    cardid_features = pd.DataFrame() \n    cardid_features['card_id'] = df['card_id'].unique()\n    \n    df['month_diff'] = (pd.to_timedelta(datetime.datetime.today() - df['purchase_date']).dt.days)//30 \n    df['month_diff'] = df['month_diff'].astype(int)\n    df['month_diff'] += df['month_lag']\n    \n    print( '*' * 30, 'Traditional Features', '*' * 30)\n    ht_card_id_gp = df.groupby('card_id')\n    print(' card id : count')\n    cardid_features['card_id_cnt'] = ht_card_id_gp['authorized_flag'].count().values\n    cardid_features['card_id_isau_mean'] = ht_card_id_gp['authorized_flag'].mean().values \n    cardid_features['card_id_isau_sum'] = ht_card_id_gp['authorized_flag'].sum().values\n    cardid_features['month_diff_mean'] = ht_card_id_gp['month_diff'].mean().values\n\n    print('card id(city_id,installments,merchant_category_id,.......): nunique, cnt/nunique')  \n    for col in tqdm_notebook(['state_id','city_id','installments','merchant_id', 'merchant_category_id','purchase_date_floorday']): \n        cardid_features['card_id_%s_nunique'%col] = ht_card_id_gp[col].nunique().values \n        cardid_features['card_id_cnt_divide_%s_nunique'%col] = cardid_features['card_id_cnt'].values / cardid_features['card_id_%s_nunique'%col].values \n    \n    for col in tqdm_notebook(['purchase_amount','purchase_amount_ddgd_98','purchase_amount_wdgd_96','purchase_amount_mdgd_90','purchase_amount_mdgd_80']): #,'purchase_amount_ddgd_98','purchase_amount_ddgd_99','purchase_amount_wdgd_96','purchase_amount_wdgd_97','purchase_amount_mdgd_90','purchase_amount_mdgd_80']): \n        if col =='purchase_amount': \n            for opt in ['sum','mean','std','median']: \n                cardid_features['card_id_' +col+ '_' + opt] = ht_card_id_gp[col].agg(opt).values   \n        else: \n            for opt in ['sum']: \n                cardid_features['card_id_' +col+ '_' + opt] = ht_card_id_gp[col].agg(opt).values  \n\n    print( '*' * 30, 'Pivot Features', '*' * 30) \n    print('Count  Pivot') \n    for pivot_col in tqdm_notebook(['category_1','category_2','category_3','month_lag','subsector_id','weekend']): #'city_id',  \n        tmp = df.groupby(['card_id',pivot_col])['merchant_id'].count().to_frame(pivot_col + '_count') \n        tmp.reset_index(inplace =True)   \n               \n        tmp_pivot = pd.pivot_table(data=tmp,index = 'card_id',columns=pivot_col,values=pivot_col + '_count',fill_value=0) \n        tmp_pivot.columns = [tmp_pivot.columns.names[0] + '_cnt_pivot_'+ str(col) for col in tmp_pivot.columns] \n        tmp_pivot.reset_index(inplace = True) \n        cardid_features = cardid_features.merge(tmp_pivot, on = 'card_id', how='left') \n            \n        if  pivot_col!='weekend' and  pivot_col!='installments': \n            tmp = df.groupby(['card_id',pivot_col])['purchase_date_floorday'].nunique().to_frame(pivot_col + '_purchase_date_floorday_nunique')  \n            tmp1 = df.groupby(['card_id'])['purchase_date_floorday'].nunique().to_frame('purchase_date_floorday_nunique')  \n            tmp.reset_index(inplace =True)   \n            tmp1.reset_index(inplace =True)    \n            tmp = tmp.merge(tmp1, on ='card_id', how='left') \n            tmp[pivot_col + '_day_nunique_pct'] = tmp[pivot_col + '_purchase_date_floorday_nunique'].values / tmp['purchase_date_floorday_nunique'].values \n               \n            tmp_pivot = pd.pivot_table(data=tmp,index = 'card_id',columns=pivot_col,values=pivot_col + '_day_nunique_pct',fill_value=0) \n            tmp_pivot.columns = [tmp_pivot.columns.names[0] + '_day_nunique_pct_'+ str(col) for col in tmp_pivot.columns] \n            tmp_pivot.reset_index(inplace = True) \n            cardid_features = cardid_features.merge(tmp_pivot, on = 'card_id', how='left') \n      \n    cardid_features = reduce_mem_usage(cardid_features)     \n    new_col_names = [] \n    for col in cardid_features.columns: \n        if col == 'card_id': \n            new_col_names.append(col) \n        else: \n            new_col_names.append(prefix + col) \n    cardid_features.columns = new_col_names \n    return cardid_features  \n\nhist_cardf_tran_last2 = get_cardf_tran_last2(historical_transactions, month = 3, prefix = 'hist_last2_') ","metadata":{"execution":{"iopub.status.busy":"2024-09-17T23:30:51.400849Z","iopub.execute_input":"2024-09-17T23:30:51.401154Z","iopub.status.idle":"2024-09-17T23:32:07.302513Z","shell.execute_reply.started":"2024-09-17T23:30:51.401122Z","shell.execute_reply":"2024-09-17T23:32:07.301409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Second-order cross-feature derivation\n\ndef successive_aggregates(df_, prefix = 'levelAB_'):\n    df = df_.copy()\n    cardid_features = pd.DataFrame()\n    cardid_features['card_id'] = df['card_id'].unique()\n    \n    level12_nunique = [('month_lag','state_id'),('month_lag','city_id'),('month_lag','subsector_id'),('month_lag','merchant_category_id'),('month_lag','merchant_id'),('month_lag','purchase_date_floorday'), \n                            ('subsector_id','merchant_category_id'),('subsector_id','merchant_id'),('subsector_id','purchase_date_floorday'),('subsector_id','month_lag'), \n                            ('merchant_category_id', 'merchant_id'),('merchant_category_id','purchase_date_floorday'),('merchant_category_id','month_lag'), \n                            ('purchase_date_floorday', 'merchant_id'),('purchase_date_floorday','merchant_category_id'),('purchase_date_floorday','subsector_id')]     \n    \n    for col_level1,col_level2 in tqdm_notebook(level12_nunique):   \n        level1 = df.groupby(['card_id',col_level1])[col_level2].nunique().to_frame(col_level2 + '_nunique') \n        level1.reset_index(inplace =True)   \n        level2 = level1.groupby('card_id')[col_level2 + '_nunique'].agg(['mean', 'max', 'std']) \n        level2 = pd.DataFrame(level2) \n        level2.columns = [col_level1 + '_' + col_level2 + '_nunique_' + col for col in level2.columns.values] \n        level2.reset_index(inplace = True) \n        cardid_features = cardid_features.merge(level2, on='card_id', how='left')  \n        \n    level12_count = ['month_lag','state_id','city_id','subsector_id','merchant_category_id','merchant_id','purchase_date_floorday'] \n    \n    for col_level in tqdm_notebook(level12_count):  \n        level1 = df.groupby(['card_id',col_level])['merchant_id'].count().to_frame(col_level + '_count') \n        level1.reset_index(inplace =True)   \n        level2 = level1.groupby('card_id')[col_level + '_count'].agg(['mean', 'max', 'std']) \n        level2 = pd.DataFrame(level2) \n        level2.columns = [col_level + '_count_' + col for col in level2.columns.values] \n        level2.reset_index(inplace = True) \n        cardid_features = cardid_features.merge(level2, on='card_id', how='left')  \n    \n    level12_meansum = [('month_lag','purchase_amount'),('state_id','purchase_amount'),('city_id','purchase_amount'),('subsector_id','purchase_amount'), \n                            ('merchant_category_id','purchase_amount'),('merchant_id','purchase_amount'),('purchase_date_floorday','purchase_amount')] \n\n    for col_level1,col_level2 in tqdm_notebook(level12_meansum):  \n        level1 = df.groupby(['card_id',col_level1])[col_level2].sum().to_frame(col_level2 + '_sum') \n        level1.reset_index(inplace =True)   \n        level2 = level1.groupby('card_id')[col_level2 + '_sum'].agg(['mean', 'max', 'std']) \n        level2 = pd.DataFrame(level2) \n        level2.columns = [col_level1 + '_' + col_level2 + '_sum_' + col for col in level2.columns.values] \n        level2.reset_index(inplace = True) \n        cardid_features = cardid_features.merge(level2, on='card_id', how='left')  \n    \n    cardid_features = reduce_mem_usage(cardid_features) \n    new_col_names = [] \n    for col in cardid_features.columns: \n        if col == 'card_id': \n            new_col_names.append(col) \n        else: \n            new_col_names.append(prefix + col) \n    cardid_features.columns = new_col_names \n    return cardid_features   \n\nprint('hist...') \nhist_levelAB = successive_aggregates(historical_transactions, prefix = 'hist_levelAB_') ","metadata":{"execution":{"iopub.status.busy":"2024-09-17T23:32:07.303937Z","iopub.execute_input":"2024-09-17T23:32:07.304273Z","iopub.status.idle":"2024-09-17T23:37:44.080110Z","shell.execute_reply.started":"2024-09-17T23:32:07.304239Z","shell.execute_reply":"2024-09-17T23:37:44.078985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Merge the features\n\nprint(train.shape)\nprint(test.shape)\n\n## merge to train/test set\nprint('#_____basic statistical features____#')\ntrain = pd.merge(train, auth_base_stat, on='card_id', how='left') \ntest = pd.merge(test, auth_base_stat, on='card_id', how='left') \ntrain = pd.merge(train, hist_base_stat, on='card_id', how='left') \ntest = pd.merge(test, hist_base_stat, on='card_id', how='left') \ntrain = pd.merge(train, new_base_stat , on='card_id', how='left') \ntest = pd.merge(test, new_base_stat , on='card_id', how='left') \nprint(train.shape)\nprint(test.shape)\nprint('#_____cardid features____#')\ntrain = pd.merge(train, auth_cardf_tran, on='card_id', how='left')\ntest = pd.merge(test, auth_cardf_tran, on='card_id', how='left')\ntrain = pd.merge(train, hist_cardf_tran, on='card_id', how='left')\ntest = pd.merge(test, hist_cardf_tran, on='card_id', how='left')\ntrain = pd.merge(train, new_cardf_tran , on='card_id', how='left')\ntest = pd.merge(test, new_cardf_tran , on='card_id', how='left') \nprint(train.shape)\nprint(test.shape)\nprint('#_____last 2 months cardid features____#')\ntrain = pd.merge(train, hist_cardf_tran_last2, on='card_id', how='left') \ntest = pd.merge(test, hist_cardf_tran_last2, on='card_id', how='left') \nprint(train.shape)\nprint(test.shape)\nprint('#_____second order features____#')\ntrain = pd.merge(train, hist_levelAB, on='card_id', how='left')\ntest = pd.merge(test, hist_levelAB, on='card_id', how='left') \nprint(train.shape)\nprint(test.shape)\n\n# Generate some features by elementary arithmetic\ntrain['outliers'] = 0\ntrain.loc[train['target'] < -30, 'outliers'] = 1 \ntrain['outliers'].value_counts()\nfor f in ['feature_1','feature_2','feature_3']:\n    colname = f+'_outliers_mean'\n    order_label = train.groupby([f])['outliers'].mean() \n    for df in [train, test]:\n        df[colname] = df[f].map(order_label) \n\nfor df in [train, test]:\n    df['days_feature1'] = df['elapsed_time'] * df['feature_1'] \n    df['days_feature2'] = df['elapsed_time'] * df['feature_2'] \n    df['days_feature3'] = df['elapsed_time'] * df['feature_3']\n    df['days_feature1_ratio'] = df['feature_1'] / df['elapsed_time'] \n    df['days_feature2_ratio'] = df['feature_2'] / df['elapsed_time'] \n    df['days_feature3_ratio'] = df['feature_3'] / df['elapsed_time']\n    df['feature_sum'] = df['feature_1'] + df['feature_2'] + df['feature_3']\n    \n    df['feature_mean'] = df['feature_sum']/3\n    df['feature_max'] = df[['feature_1', 'feature_2', 'feature_3']].max(axis=1) \n    df['feature_min'] = df[['feature_1', 'feature_2', 'feature_3']].min(axis=1) \n    df['feature_var'] = df[['feature_1', 'feature_2', 'feature_3']].std(axis=1)\n    \n    df['card_id_total'] = df['hist_card_id_size']+df['new_card_id_size'] \n    df['card_id_cnt_total'] = df['hist_card_id_count']+df['new_card_id_count'] \n    df['card_id_cnt_ratio'] = df['new_card_id_count']/df['hist_card_id_count'] \n    df['purchase_amount_total'] = df['hist_cardf_card_id_purchase_amount_sum']+df['new_cardf_card_id_purchase_amount_sum'] \n    df['purchase_amount_ratio'] = df['new_cardf_card_id_purchase_amount_sum']/df['hist_cardf_card_id_purchase_amount_sum'] \n    df['month_diff_ratio'] = df['new_cardf_month_diff_mean']/df['hist_cardf_month_diff_mean'] \n    df['installments_total'] = df['new_cardf_card_id_installments_sum']+df['auth_cardf_card_id_installments_sum'] \n    df['installments_ratio'] = df['new_cardf_card_id_installments_sum']/df['auth_cardf_card_id_installments_sum'] \n    df['price_total'] = df['purchase_amount_total']/df['installments_total'] \n    df['new_CLV'] = df['new_card_id_count'] * df['new_cardf_card_id_purchase_amount_sum'] / df['new_cardf_month_diff_mean'] \n    df['hist_CLV'] = df['hist_card_id_count'] * df['hist_cardf_card_id_purchase_amount_sum'] / df['hist_cardf_month_diff_mean'] \n    df['CLV_ratio'] = df['new_CLV'] / df['hist_CLV'] ","metadata":{"execution":{"iopub.status.busy":"2024-09-17T23:37:44.081452Z","iopub.execute_input":"2024-09-17T23:37:44.081801Z","iopub.status.idle":"2024-09-17T23:37:56.516689Z","shell.execute_reply.started":"2024-09-17T23:37:44.081756Z","shell.execute_reply":"2024-09-17T23:37:56.515725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####  feature selection  ####\n\ndel_cols = []\nfor col in train.columns:\n    if 'subsector_id_cnt_' in col and 'new_cardf': \n        del_cols.append(col)\ndel_cols1 = []\nfor col in train.columns:\n    if 'subsector_id_cnt_' in col and 'hist_last2_' in col: \n        del_cols1.append(col)\ndel_cols2 = []\nfor col in train.columns:\n    if 'subsector_id_cnt_' in col and 'auth_cardf' in col: \n        del_cols2.append(col)\ndel_cols3 = []\nfor col in train.columns:\n    if 'merchant_category_id_month_lag_nunique_' in col and '_pivot_supp' in col: \n        del_cols3.append(col)\n    if 'city_id' in col and '_pivot_supp' in col: \n        del_cols3.append(col)\n    if 'month_diff' in col and 'hist_last2_' in col: \n        del_cols3.append(col)\n    if 'month_diff_std' in col or 'month_diff_gap' in col: \n        del_cols3.append(col)\n        \nfea_cols = [col for col in train.columns if train[col].dtypes!='object' and train[col].dtypes != '<M8[ns]' and col!='target' not in col and col!='min_num' \n                 and col not in del_cols and col not in del_cols1 and col not in del_cols2 and col!='target1' and col!='card_id_cnt_ht_pivot_supp'  and col not in del_cols3]    \nprint('Before delete:',train.shape[1]) \nprint('After delete:',len(fea_cols)) \n      \ntrain = train[fea_cols+['target']] \nfea_cols.remove('outliers') \ntest = test[fea_cols] \n      \ntrain.to_csv('/kaggle/working/all_train_features.csv',index=False) \ntest.to_csv('/kaggle/working/all_test_features.csv',index=False) ","metadata":{"execution":{"iopub.status.busy":"2024-09-17T23:37:56.517897Z","iopub.execute_input":"2024-09-17T23:37:56.518208Z","iopub.status.idle":"2024-09-17T23:41:02.242932Z","shell.execute_reply.started":"2024-09-17T23:37:56.518176Z","shell.execute_reply":"2024-09-17T23:41:02.242046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####  Training  ####\n\n# load all features\ntrain = pd.read_csv('/kaggle/working/all_train_features.csv')\ntest = pd.read_csv('/kaggle/working/all_test_features.csv')\n\n## get index without outliers\nnormal_index = train[train['outliers'] == 0].index.tolist() \n## training set without outliers \nntrain = train[train['outliers'] == 0] \n\ninf_cols = ['new_cardf_card_id_cnt_divide_installments_nunique', 'hist_last2_card_id_cnt_divide_installments_nunique'] \ntrain[inf_cols] = train[inf_cols].replace(np.inf, train[inf_cols].replace(np.inf, -99).max().max()) \nntrain[inf_cols] = ntrain[inf_cols].replace(np.inf, ntrain[inf_cols].replace(np.inf, -99).max().max()) \ntest[inf_cols] = test[inf_cols].replace(np.inf, test[inf_cols].replace(np.inf, -99).max().max()) \n      \ntarget = train['target'].values \nntarget = ntrain['target'].values \ntarget_binary = train['outliers'].values \n\nX_ntrain = ntrain[fea_cols].values \nX_train = train[fea_cols].values \nX_test = test[fea_cols].values \n\ny_train = target \ny_ntrain = ntarget \ny_train_binary = target_binary \n      \nprint('train:',train.shape) \nprint('ntrain:',ntrain.shape)\nprint('test:', test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-09-17T23:59:31.205964Z","iopub.execute_input":"2024-09-17T23:59:31.207142Z","iopub.status.idle":"2024-09-17T23:59:32.065538Z","shell.execute_reply.started":"2024-09-17T23:59:31.207099Z","shell.execute_reply":"2024-09-17T23:59:32.064536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function defined to complete the training process for all models:\n# To call the three different models, and to do both classification and regression prediction.\n\ndef train_model(X, X_test, y, params, folds, model_type='lgb', eval_type='regression'):\n    oof = np.zeros(X.shape[0])\n    predictions = np.zeros(X_test.shape[0])\n    scores = []\n    for fold_n, (trn_idx, val_idx) in enumerate(folds.split(X, y)): \n        print('Fold', fold_n, 'started at', time.ctime())\n        if model_type == 'lgb':\n            trn_data = lgb.Dataset(X[trn_idx], y[trn_idx])\n            val_data = lgb.Dataset(X[val_idx], y[val_idx])\n            clf = lgb.train(params, trn_data, num_boost_round=20000, \n                            valid_sets=[trn_data, val_data], \n                            callbacks=[lgb.log_evaluation(100), lgb.early_stopping(300)])\n            oof[val_idx] = clf.predict(X[val_idx], num_iteration=clf.best_iteration)\n            predictions += clf.predict(X_test, num_iteration=clf.best_iteration) / folds.n_splits\n            \n        if model_type == 'xgb': \n            trn_data = xgb.DMatrix(X[trn_idx], y[trn_idx]) \n            val_data = xgb.DMatrix(X[val_idx], y[val_idx]) \n            watchlist = [(trn_data, 'train'), (val_data, 'valid_data')] \n            clf = xgb.train(dtrain=trn_data, num_boost_round=20000,\n                            evals=watchlist, early_stopping_rounds=200,\n                            verbose_eval=100, params=params) \n            oof[val_idx] = clf.predict(xgb.DMatrix(X[val_idx])) \n            predictions += clf.predict(xgb.DMatrix(X_test)) / folds.n_splits \n        \n        if (model_type == 'cat') and (eval_type == 'regression'): \n            clf = CatBoostRegressor(iterations=20000, eval_metric='RMSE', **params) \n            clf.fit(X[trn_idx], y[trn_idx], \n                    eval_set=(X[val_idx], y[val_idx]), \n                    cat_features=[], use_best_model=True, verbose=100) \n            oof[val_idx] = clf.predict(X[val_idx]) \n            predictions += clf.predict(X_test) / folds.n_splits \n            \n        if (model_type == 'cat') and (eval_type == 'binary'): \n            clf = CatBoostClassifier(iterations=20000, eval_metric='Logloss', **params) \n            clf.fit(X[trn_idx], y[trn_idx], \n                    eval_set=(X[val_idx], y[val_idx]), \n                    cat_features=[], use_best_model=True, verbose=100) \n            oof[val_idx] = clf.predict_proba(X[val_idx])[:,1] \n            predictions += clf.predict_proba(X_test)[:,1] / folds.n_splits \n            \n        print(predictions) \n        if eval_type == 'regression': \n            scores.append(mean_squared_error(oof[val_idx], y[val_idx])**0.5) \n        if eval_type == 'binary': \n            scores.append(log_loss(y[val_idx], oof[val_idx])) \n\n    print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores))) \n    return oof, predictions, scores","metadata":{"execution":{"iopub.status.busy":"2024-09-18T00:11:56.180417Z","iopub.execute_input":"2024-09-18T00:11:56.181355Z","iopub.status.idle":"2024-09-18T00:11:56.195967Z","shell.execute_reply.started":"2024-09-18T00:11:56.181311Z","shell.execute_reply":"2024-09-18T00:11:56.194824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####  Train LightGBM Model  ####\n\nlgb_params = {'num_leaves': 63, \n              'min_data_in_leaf': 32,  \n              'objective':'regression', \n              'max_depth': -1, \n              'learning_rate': 0.01, \n              \"min_child_samples\": 20, \n              \"boosting\": \"gbdt\", \n              \"feature_fraction\": 0.9, \n              \"bagging_freq\": 1, \n              \"bagging_fraction\": 0.9 , \n              \"bagging_seed\": 11, \n              \"metric\": 'rmse', \n              \"lambda_l1\": 0.1, \n              \"verbosity\": -1}\nfolds = KFold(n_splits=5, shuffle=True, random_state=4096) \nprint('#'*10,'Regression','#'*10) \noof_lgb, predictions_lgb, scores_lgb = train_model(X_train , X_test, y_train, params=lgb_params, folds=folds, model_type='lgb', eval_type='regression') \nprint('#'*10,'regression without outliers','#'*10) \noof_nlgb, predictions_nlgb, scores_nlgb = train_model(X_ntrain, X_test, y_ntrain, params=lgb_params, folds=folds, model_type='lgb', eval_type='regression') \nprint('#'*10,'Classification','#'*10) \nlgb_params['objective'] = 'binary' \nlgb_params['metric'] = 'binary_logloss' \noof_blgb, predictions_blgb, scores_blgb = train_model(X_train , X_test, y_train_binary, params=lgb_params, folds=folds, model_type='lgb', eval_type='binary')\n\n# Save the prediction results for submission\nsub_df = pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/sample_submission.csv') \nsub_df[\"target\"] = predictions_lgb \nsub_df.to_csv('/kaggle/working/submission_lgb.csv', index=False)\noof_lgb = pd.DataFrame(oof_lgb) \noof_nlgb = pd.DataFrame(oof_nlgb) \noof_blgb = pd.DataFrame(oof_blgb)\npredictions_lgb = pd.DataFrame(predictions_lgb) \npredictions_nlgb = pd.DataFrame(predictions_nlgb) \npredictions_blgb = pd.DataFrame(predictions_blgb)\noof_lgb.to_csv('/kaggle/working/oof_lgb.csv',header=None,index=False) \noof_blgb.to_csv('/kaggle/working/oof_blgb.csv',header=None,index=False) \noof_nlgb.to_csv('/kaggle/working/oof_nlgb.csv',header=None,index=False)\npredictions_lgb.to_csv('/kaggle/working/predictions_lgb.csv',header=None,index=False) \npredictions_nlgb.to_csv('/kaggle/working/predictions_nlgb.csv',header=None,index=False) \npredictions_blgb.to_csv('/kaggle/working/predictions_blgb.csv',header=None,index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####  Train XGBoost Model  ####\nxgb_params = {'eta': 0.05, \n              'max_leaves': 47, \n              'max_depth': 10, \n              'subsample': 0.8, \n              'colsample_bytree': 0.5,\n              'min_child_weight': 40, \n              'max_bin': 128, \n              'reg_alpha': 2.0, \n              'reg_lambda': 2.0, \n              'objective': 'reg:linear', \n              'eval_metric': 'rmse', \n              'silent': True, \n              'nthread': 4}\nfolds = KFold(n_splits=5, shuffle=True, random_state=4096)\nprint('#'*10,'Regression','#'*10)\noof_xgb , predictions_xgb , scores_xgb = train_model(X_train , X_test, y_train , params=xgb_params, folds=folds, model_type='xgb', eval_type='regression') \nprint('#'*10,'Regression without outliers','#'*10)\noof_nxgb, predictions_nxgb, scores_nxgb = train_model(X_ntrain, X_test, y_ntrain, params=xgb_params, folds=folds, model_type='xgb', eval_type='regression') \nprint('#'*10,'Classification','#'*10)\nxgb_params['objective'] = 'binary:logistic'\nxgb_params['metric'] = 'binary_logloss'\noof_bxgb, predictions_bxgb, scores_bxgb = train_model(X_train , X_test, y_train_binary, params=xgb_params, folds=folds, model_type='xgb', eval_type='binary')\n\n# Save the prediction results for submission\nsub_df = pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/sample_submission.csv') \nsub_df[\"target\"] = predictions_xgb \nsub_df.to_csv('/kaggle/working/submission_xgb.csv', index=False)\noof_xgb = pd.DataFrame(oof_xgb) \noof_nxgb = pd.DataFrame(oof_nxgb) \noof_bxgb = pd.DataFrame(oof_bxgb)\npredictions_xgb = pd.DataFrame(predictions_xgb) \npredictions_nxgb = pd.DataFrame(predictions_nxgb) \npredictions_bxgb = pd.DataFrame(predictions_bxgb)\noof_xgb.to_csv('/kaggle/working/oof_xgb.csv',header=None,index=False) \noof_bxgb.to_csv('/kaggle/working/oof_bxgb.csv',header=None,index=False) \noof_nxgb.to_csv('/kaggle/working/oof_nxgb.csv',header=None,index=False)\npredictions_xgb.to_csv('/kaggle/working/predictions_xgb.csv',header=None,index=False) \npredictions_nxgb.to_csv('/kaggle/working/predictions_nxgb.csv',header=None,index=False) \npredictions_bxgb.to_csv('/kaggle/working/predictions_bxgb.csv',header=None,index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####  Train CatBoost Model  ####\ncat_params = {'learning_rate': 0.05, \n              'depth': 9, \n              'l2_leaf_reg': 10, \n              'bootstrap_type': 'Bernoulli', \n              'od_type': 'Iter', \n              'od_wait': 50, \n              'random_seed': 4096, \n              'allow_writing_files': False}\nfolds = KFold(n_splits=5, shuffle=True, random_state=4096)\nprint('#'*10,'Regression','#'*10)\noof_cat , predictions_cat , scores_cat = train_model(X_train , X_test, y_train , params=cat_params, folds=folds, model_type='cat', eval_type='regression') \nprint('#'*10,'Regression without outliers','#'*10)\noof_ncat, predictions_ncat, scores_ncat = train_model(X_ntrain, X_test, y_ntrain, params=cat_params, folds=folds, model_type='cat', eval_type='regression') \nprint('#'*10,'Classification','#'*10)\noof_bcat, predictions_bcat, scores_bcat = train_model(X_train , X_test, y_train_binary, params=cat_params, folds=folds, model_type='cat', eval_type='binary')\n\n# Save the prediction results for submission\nsub_df = pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/sample_submission.csv') \nsub_df[\"target\"] = predictions_cat \nsub_df.to_csv('/kaggle/working/submission_cat.csv', index=False)\noof_cat = pd.DataFrame(oof_cat) \noof_ncat = pd.DataFrame(oof_ncat) \noof_bcat = pd.DataFrame(oof_bcat)\npredictions_cat = pd.DataFrame(predictions_cat) \npredictions_ncat = pd.DataFrame(predictions_ncat) \npredictions_bcat = pd.DataFrame(predictions_bcat)\noof_cat.to_csv('/kaggle/working/oof_cat.csv',header=None,index=False) \noof_bcat.to_csv('/kaggle/working/oof_bcat.csv',header=None,index=False) \noof_ncat.to_csv('/kaggle/working/oof_ncat.csv',header=None,index=False)\npredictions_cat.to_csv('/kaggle/working/predictions_cat.csv',header=None,index=False) \npredictions_ncat.to_csv('/kaggle/working/predictions_ncat.csv',header=None,index=False) \npredictions_bcat.to_csv('/kaggle/working/predictions_bcat.csv',header=None,index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####  Voting  ####\nsub_df = pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/sample_submission.csv')\nsub_df[\"target\"] = (predictions_lgb + predictions_xgb.values + predictions_cat.values) / 3\nsub_df.to_csv('/kaggle/working/submission_voting.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####  Stacking  ####\ndef stack_model(oof_1, oof_2, oof_3, predictions_1, predictions_2, predictions_3, y, eval_type='regression'): \n    train_stack = np.hstack([oof_1, oof_2, oof_3])\n    test_stack = np.hstack([predictions_1, predictions_2, predictions_3])\n    oof = np.zeros(train_stack.shape[0])\n    predictions = np.zeros(test_stack.shape[0])\n    \n    folds = RepeatedKFold(n_splits=5, n_repeats=2, random_state=4096)\n    \n    for fold_, (trn_idx, val_idx) in enumerate(folds.split(train_stack, y)): \n        print(\"fold #{}\".format(fold_+1))\n        trn_data, trn_y = train_stack[trn_idx], y[trn_idx] \n        val_data, val_y = train_stack[val_idx], y[val_idx] \n        print(\"#\" * 10 + \"Stacking \" + str(fold_+1) + \"#\" * 10) \n        clf = BayesianRidge()\n        clf.fit(trn_data, trn_y)\n        predictions += clf.predict(test_stack) / (5 * 2)\n        \n    if eval_type == 'regression':\n        print('mean: ', np.sqrt(mean_squared_error(y, oof)))\n    if eval_type == 'binary': \n        print('mean: ', log_loss(y, oof))\n    return oof, predictions\n\nprint('#'*30)\noof_stack , predictions_stack = stack_model(oof_lgb, oof_xgb, oof_cat, predictions_lgb, predictions_xgb, predictions_cat, y_train, eval_type='regression')\nprint('#'*30)\noof_nstack, predictions_nstack = stack_model(oof_nlgb, oof_nxgb, oof_ncat, predictions_nlgb, predictions_nxgb, predictions_ncat, y_ntrain, eval_type='regression')\nprint('#'*30)\noof_bstack, predictions_bstack = stack_model(oof_blgb, oof_bxgb, oof_bcat, predictions_blgb, predictions_bxgb, predictions_bcat, y_train_binary, eval_type='binary')\n                                             \nsub_df = pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/sample_submission.csv') \nsub_df[\"target\"] = predictions_stack \nsub_df.to_csv('/kaggle/working/submission_stack.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####  Trick  ####\nsub_df = pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/sample_submission.csv')\nsub_df[\"target\"] = predictions_bstack*-33.219281 + (1-predictions_bstack)*predictions_nstack \nsub_df.to_csv('/kaggle/working/submission_stack&trick.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####  Trick&voting  ####\nsub_df = pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/sample_submission.csv')\nsub_df[\"target\"] = (predictions_bstack*-33.219281 + (1-predictions_bstack)*predictions_nstack)*0.5 + predictions_stack*0.5\nsub_df.to_csv('/kaggle/working/submission_stack&trick&voting.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]}]}