{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install optuna","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import optuna\noptuna.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"185ea893b6604e7653b8e259bbee7fcd24f560f5"},"cell_type":"code","source":"# coding: utf-8\nimport gc\nimport os\nimport time\nimport datetime\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb_origin\nimport optuna.integration.lightgbm as lgb\nfrom pathlib import Path\nfrom tqdm import tqdm_notebook as tqdm\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_path = Path(\"../input\")\ntrain_path = input_path / \"train.csv\"\ntest_path = input_path / \"test.csv\"\nhistorical_tr_path = input_path / \"historical_transactions.csv\"\nnew_merchant_tr_path = input_path / \"new_merchant_transactions.csv\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bb1c63c5e3ff5f9874229616eb4dd59c552c1dd1"},"cell_type":"markdown","source":"# 1. Loading the data"},{"metadata":{"trusted":true,"_uuid":"85018ae13c836df8d991f36e8409672a2758065f"},"cell_type":"code","source":"train = pd.read_csv(train_path, parse_dates=[\"first_active_month\"])\ntest = pd.read_csv(test_path, parse_dates=[\"first_active_month\"])\nnew_merchant = pd.read_csv(new_merchant_tr_path, parse_dates=['purchase_date'])\nht = pd.read_csv(historical_tr_path, parse_dates=['purchase_date'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8956542df13b76622f346ddd3266b18f3a8c1c7"},"cell_type":"code","source":"for df in [train,test]:\n    df['start_year'] = df['first_active_month'].dt.year\n    df['start_month'] = df['first_active_month'].dt.month\n    df['elapsed_time'] = (datetime.date(2018, 2, 1) - df['first_active_month'].dt.date).dt.days\n\nytrain = train['target']\ndel train['target']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ae9fd905aa7ad1b5f2689a70c3b4845da80d08ba"},"cell_type":"markdown","source":"# Historical data"},{"metadata":{"trusted":true,"_uuid":"0b9f8bc1cec14070d011bde41dec3a5b63fe0dad"},"cell_type":"code","source":"# binarize the categorical variables where it makes sense\nht['authorized_flag'] = ht['authorized_flag'].map({'Y':1, 'N':0})\nht['category_1'] = ht['category_1'].map({'Y':1, 'N':0})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"103c94e2c82cd38425dc88e4867b5dd06def5174"},"cell_type":"code","source":"ht['category_2x1'] = (ht['category_2'] == 1) + 0\nht['category_2x2'] = (ht['category_2'] == 2) + 0\nht['category_2x3'] = (ht['category_2'] == 3) + 0\nht['category_2x4'] = (ht['category_2'] == 4) + 0\nht['category_2x5'] = (ht['category_2'] == 5) + 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02cc8a1936790fc71c63e8fff0720b92e0783d07"},"cell_type":"code","source":"ht['category_3A'] = (ht['category_3'].astype(str) == 'A') + 0\nht['category_3B'] = (ht['category_3'].astype(str) == 'B') + 0\nht['category_3C'] = (ht['category_3'].astype(str) == 'C') + 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1d1b3193d2403ba707cc275987d755604f6eeac"},"cell_type":"code","source":"def aggregate_historical_transactions(history):\n    \n    history.loc[:, 'purchase_date'] = pd.DatetimeIndex(history['purchase_date']).\\\n                                      astype(np.int64) * 1e-9\n    \n    agg_func = {\n        'authorized_flag': ['sum', 'mean'],\n        'category_1': ['sum', 'mean'],\n        'category_2': ['nunique'],\n        'category_3A': ['sum'],\n        'category_3B': ['sum'],\n        'category_3C': ['sum'],\n        'category_2x1': ['sum','mean'],\n        'category_2x2': ['sum','mean'],\n        'category_2x3': ['sum','mean'],\n        'category_2x4': ['sum','mean'],\n        'category_2x5': ['sum','mean'],        \n        'city_id': ['nunique'],\n        'installments': ['sum', 'median', 'max', 'min', 'std'],\n        'merchant_category_id': ['nunique'],\n        'merchant_id': ['nunique'],\n        'month_lag': ['min', 'max'],\n        'purchase_amount': ['sum', 'median', 'max', 'min', 'std'],\n        'purchase_date': [np.ptp, 'max', 'min'],\n        'state_id': ['nunique'],\n        'subsector_id': ['nunique'],\n\n        }\n    agg_history = history.groupby(['card_id']).agg(agg_func)\n    agg_history.columns = ['hist_' + '_'.join(col).strip() \n                           for col in agg_history.columns.values]\n    agg_history.reset_index(inplace=True)\n    \n    df = (history.groupby('card_id')\n          .size()\n          .reset_index(name='hist_transactions_count'))\n    \n    agg_history = pd.merge(df, agg_history, on='card_id', how='left')\n    \n    return agg_history\n\nhistory = aggregate_historical_transactions(ht)\n\ndel ht\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9a3f60926c40fef156000842eada5ee4717d7311"},"cell_type":"markdown","source":"#New data"},{"metadata":{"trusted":true,"_uuid":"35eb8d06c99e5bb46555a23c2e903ac274778057"},"cell_type":"code","source":"new_merchant['authorized_flag'] = new_merchant['authorized_flag'].map({'Y':1, 'N':0})\nnew_merchant['category_1'] = new_merchant['category_1'].map({'Y':1, 'N':0})\nnew_merchant['category_3A'] = (new_merchant['category_3'].astype(str) == 'A') + 0\nnew_merchant['category_3B'] = (new_merchant['category_3'].astype(str) == 'B') + 0\nnew_merchant['category_3C'] = (new_merchant['category_3'].astype(str) == 'C') + 0\n\nnew_merchant['category_2x1'] = (new_merchant['category_2'] == 1) + 0\nnew_merchant['category_2x2'] = (new_merchant['category_2'] == 2) + 0\nnew_merchant['category_2x3'] = (new_merchant['category_2'] == 3) + 0\nnew_merchant['category_2x4'] = (new_merchant['category_2'] == 4) + 0\nnew_merchant['category_2x5'] = (new_merchant['category_2'] == 5) + 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a89eed26a836c7423ce9ad5db260816d89a7bee7"},"cell_type":"code","source":"new_merchant['purchase_date'] = pd.DatetimeIndex(new_merchant['purchase_date']).\\\n                                      astype(np.int64) * 1e-9","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d8d510d88deeca70bc06c239c4c347c69266829"},"cell_type":"code","source":"def aggregate_new_transactions(new_trans):    \n    \n    \n    agg_func = {\n        'authorized_flag': ['sum', 'mean'],\n        'category_1': ['sum', 'mean'],\n        'category_2': ['nunique'],\n        'category_3A': ['sum'],\n        'category_3B': ['sum'],\n        'category_3C': ['sum'],     \n        'category_2x1': ['sum','mean'],\n        'category_2x2': ['sum','mean'],\n        'category_2x3': ['sum','mean'],\n        'category_2x4': ['sum','mean'],\n        'category_2x5': ['sum','mean'],        \n\n        'city_id': ['nunique'],\n        'installments': ['sum', 'median', 'max', 'min', 'std'],\n        'merchant_category_id': ['nunique'],\n        'merchant_id': ['nunique'],\n        'month_lag': ['min', 'max'],\n        'purchase_amount': ['sum', 'median', 'max', 'min', 'std'],\n        'purchase_date': [np.ptp, 'max', 'min'],\n        'state_id': ['nunique'],\n        'subsector_id': ['nunique']        \n        }\n    agg_new_trans = new_trans.groupby(['card_id']).agg(agg_func)\n    agg_new_trans.columns = ['new_' + '_'.join(col).strip() \n                           for col in agg_new_trans.columns.values]\n    agg_new_trans.reset_index(inplace=True)\n    \n    df = (new_trans.groupby('card_id')\n          .size()\n          .reset_index(name='new_transactions_count'))\n    \n    agg_new_trans = pd.merge(df, agg_new_trans, on='card_id', how='left')\n    \n    return agg_new_trans\n\nnew_trans = aggregate_new_transactions(new_merchant)\n\ndel new_merchant","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"604fc7dd435cb9f1d7e3a9427efe9991be0e232a"},"cell_type":"markdown","source":"# Combine"},{"metadata":{"trusted":true,"_uuid":"238a0d10e132c839e8b07f0697957a6152b4b34a"},"cell_type":"code","source":"print(train.shape)\nprint(test.shape)\n\nxtrain = pd.merge(train, new_trans, on='card_id', how='left')\nxtest = pd.merge(test, new_trans, on='card_id', how='left')\n\ndel new_trans\n\nprint(xtrain.shape)\nprint(xtest.shape)\n\nxtrain = pd.merge(xtrain, history, on='card_id', how='left')\nxtest = pd.merge(xtest, history, on='card_id', how='left')\n\ndel history\n\nprint(xtrain.shape)\nprint(xtest.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f69b52ac820b5902a89cd18f5a7af72b05919f74"},"cell_type":"code","source":"xtrain.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0debdf6f7bc2ea3b6ce134371036b0ccbbaff00"},"cell_type":"code","source":"xtrain.drop('first_active_month', axis = 1, inplace = True)\nxtest.drop('first_active_month', axis = 1, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f464fd22fccd35b85b257f924b67f05f6eaa787"},"cell_type":"code","source":"categorical_feats = ['feature_1', 'feature_2', 'feature_3']\n\nfor col in categorical_feats:\n    print(col)\n    lbl = LabelEncoder()\n    lbl.fit(list(xtrain[col].values.astype('str')) + list(xtest[col].values.astype('str')))\n    xtrain[col] = lbl.transform(list(xtrain[col].values.astype('str')))\n    xtest[col] = lbl.transform(list(xtest[col].values.astype('str')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a071f42f0614363eb71655cf387054572052a65"},"cell_type":"code","source":"df_all = pd.concat([xtrain, xtest])\ndf_all = pd.get_dummies(df_all, columns=categorical_feats)\n\nlen_train = xtrain.shape[0]\n\nxtrain = df_all[:len_train]\nxtest = df_all[len_train:]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9a53b5f29a92afc218812b761c20f8bedece7ce8"},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true,"_uuid":"194940bdf5c95f8426320912bdcc72c40de815b3"},"cell_type":"code","source":"# prepare for modeling\nid_train = xtrain['card_id'].copy(); xtrain.drop('card_id', axis = 1, inplace = True)\nid_test = xtest['card_id'].copy(); xtest.drop('card_id', axis = 1, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_splits = 5\nfolds = KFold(n_splits=n_splits, shuffle=True, random_state=15)\noof = np.zeros(len(xtrain))\npredictions = np.zeros(len(xtest))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5810a39efff59ccc6e36df0085b058add307f000"},"cell_type":"code","source":"start = time.time()\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(xtrain.values, ytrain.values)):\n    print('Fold {}/{}'.format(fold_ + 1, folds.n_splits))\n    \n    x0,y0 = xtrain.iloc[trn_idx], ytrain[trn_idx]\n    x1,y1 = xtrain.iloc[val_idx], ytrain[val_idx]\n    \n    trn_data = lgb.Dataset(x0, label= y0)\n    val_data = lgb.Dataset(x1, label= y1)\n    \n     # LightGBMTuner\n    # Reference: https://gist.github.com/smly/367c53e855cdaeea35736f32876b7416\n    best_params = {}\n    tuning_history = []\n\n    params = {\n                'objective': 'regression',\n                'metric': 'rmse',\n            }\n    \n    num_round = 10000\n    lgb.train(\n        params,\n        trn_data,\n        num_boost_round=num_round,\n        valid_sets=[trn_data, val_data],\n        early_stopping_rounds=100,\n        verbose_eval=500,\n        best_params=best_params,\n        tuning_history=tuning_history)\n    \n    pd.DataFrame(tuning_history).to_csv('./tuning_history_{}.csv'.format(fold_ + 1))\n    best_params['learning_rate'] = 0.05\n    \n    # origin LightGBM Model\n    clf = lgb_origin.train(\n                    best_params,\n                    trn_data,\n                    num_boost_round=num_round * 2,\n                    valid_names=['train', 'valid'],\n                    valid_sets=[trn_data, val_data],\n                    verbose_eval=1000,\n                    early_stopping_rounds=1000)\n    \n    oof[val_idx] = clf.predict(xtrain.iloc[val_idx], num_iteration=clf.best_iteration)\n    \n    predictions += clf.predict(xtest, num_iteration=clf.best_iteration) / folds.n_splits\n    print(\"Fold {} CV score: {:<8.5f}\".format(fold_ + 1, mean_squared_error(ytrain[val_idx],\n                                                                            oof[val_idx])**0.5))\nprint(\"CV score: {:<8.5f}\".format(mean_squared_error(oof, ytrain)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eae4901e83078cd371a7b9e456399f9c50f0c896"},"cell_type":"code","source":"xsub = pd.DataFrame()\nxsub['card_id']  = id_test\nxsub['target'] = predictions\nxsub.to_csv('sub_lgb.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9bee266115c963fd647d6f448d5b82974e1e5d33"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}