{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nfrom sklearn.linear_model import Ridge\nimport time\nfrom sklearn import preprocessing\nimport warnings\nimport datetime\nwarnings.filterwarnings(\"ignore\")\nimport gc\nfrom tqdm import tqdm\n\nfrom scipy.stats import describe\n%matplotlib inline\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import mean_squared_error\nimport xgboost as xgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"84202acdc04bda3a9ef5043e2e9a63df1e97f0e8"},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\", parse_dates=[\"first_active_month\"])\ntest = pd.read_csv(\"../input/test.csv\", parse_dates=[\"first_active_month\"])\n\nnew_m_t=pd.read_csv(\"../input/new_merchant_transactions.csv\")\nhis_t=pd.read_csv(\"../input/historical_transactions.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c21df4156c7782bd939c8fa919db86a4dbd393fd"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8682c6bd7c389c0ab575232a296d52808986f94"},"cell_type":"code","source":"new_m_t.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b17bcd7ac4cb08440135d7750342a1a5914893da"},"cell_type":"code","source":"his_t.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8cc6b9e6add52dfd4dbbe18920443937b9e1571a"},"cell_type":"code","source":"plt.scatter(range(train.shape[0]),np.sort(train[\"target\"].values))\nplt.title(\"Scatter of Loyalty score\")\nplt.ylabel('Loyalty score', fontsize=12)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ea277f94116d5f6fcc65b78ca2412c838898d05"},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"357f8049580aa0b1bc8bfbb2f304f4b71dd2d9e2"},"cell_type":"code","source":"plt.figure(figsize=(12,8))\nsns.distplot(train[\"target\"].values, bins=150, kde=False, color=\"red\")\nplt.title(\"Histogram of Loyalty score\")\nplt.xlabel('Loyalty score', fontsize=12)\nplt.show()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6c53fb9ebcc58306305977b42c8690033b224ce"},"cell_type":"code","source":"train[train.target< -30].count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e167fbb5d1c3859365a2e340826e8616b1976a5"},"cell_type":"code","source":"sns.set_style(\"whitegrid\")\nsns.violinplot(x=train.target.values)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e7055341ce7caa3544d088914f160b73c647808e"},"cell_type":"code","source":"plt.figure(figsize=(12,5))\nplt.hist(train.feature_1.values,bins=150)\nplt.title('Histogram feature_1 counts')\nplt.xlabel('Count')\nplt.ylabel('Target')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54e19639e1d2e4b4a293f5bcd360c0353d5c266c"},"cell_type":"code","source":"plt.figure(figsize=(12,5))\nplt.hist(train.feature_2.values,bins=150)\nplt.title('Histogram feature_1 counts')\nplt.xlabel('Count')\nplt.ylabel('Target')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b6b44e8c6cf625a7fa91c341eb7fef5b142209f"},"cell_type":"code","source":"plt.figure(figsize=(12, 5))\nplt.hist(train.feature_3.values, bins=200)\nplt.title('Histogram feature_3 counts')\nplt.xlabel('Count')\nplt.ylabel('Target')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a27f290fd4acb3c5db10ea86d3b54086a562970"},"cell_type":"code","source":"train.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82c8050d8fca1692968f7a547571ef9eba12e179"},"cell_type":"code","source":"for df in [his_t,new_m_t]:\n    df['category_2'].fillna(1.0,inplace=True)\n    df['category_3'].fillna('A',inplace=True)\n    df['merchant_id'].fillna('M_ID_00a6ca8a8a',inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6534c0c82f74194b949e7f780b02d4f88caaa89f"},"cell_type":"code","source":"for df in [his_t,new_m_t]:\n    df[\"purchase_date\"]=pd.to_datetime(df[\"purchase_date\"])\n    df[\"year\"]=df[\"purchase_date\"].dt.year\n    df[\"weekofyear\"]=df[\"purchase_date\"].dt.weekofyear\n    df[\"month\"]=df[\"purchase_date\"].dt.month\n    df[\"dayofweek\"]=df[\"purchase_date\"].dt.dayofweek\n    df['weekend'] = (df.purchase_date.dt.weekday >=5).astype(int)\n    df[\"hour\"]=df[\"purchase_date\"].dt.hour\n    df[\"authorized_flag\"]=df[\"authorized_flag\"].map({\"Y\":1,\"N\":0})\n    df[\"category_1\"]=df[\"category_1\"].map({\"Y\":1,\"N\":0})\n    df[\"month_diff\"]=((datetime.datetime.today() - df['purchase_date']).dt.days)//30\n    df[\"month_diff\"]+= df['month_lag']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63e2e9202a582c3e72a777721d425956f838e9b4"},"cell_type":"code","source":"def get_new_columns(name,aggs):\n    return [name + '_' + k + '_' + agg for k in aggs.keys() for agg in aggs[k]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"931fde9b59f9cc6ab65bb95b6381a4164d01259f"},"cell_type":"code","source":"aggs = {}\nfor col in ['month','hour','weekofyear','dayofweek','year','subsector_id','merchant_id','merchant_category_id']:\n    aggs[col] = ['nunique']\n\naggs['purchase_amount'] = ['sum','max','min','mean','var']\naggs['installments'] = ['sum','max','min','mean','var']\naggs['purchase_date'] = ['max','min']\naggs['month_lag'] = ['max','min','mean','var']\naggs['month_diff'] = ['mean']\naggs['authorized_flag'] = ['sum', 'mean']\naggs['weekend'] = ['sum', 'mean']\naggs['category_1'] = ['sum', 'mean']\naggs['card_id'] = ['size']\n\nfor col in ['category_2','category_3']:\n    his_t[col+'_mean'] = his_t.groupby([col])['purchase_amount'].transform('mean')\n    aggs[col+'_mean'] = ['mean']    \n\nnew_columns = get_new_columns('hist',aggs)\nhist_trans_group = his_t.groupby('card_id').agg(aggs)\nhist_trans_group.columns = new_columns\nhist_trans_group.reset_index(drop=False,inplace=True)\nhist_trans_group['hist_purchase_date_diff'] = (hist_trans_group['hist_purchase_date_max'] - hist_trans_group['hist_purchase_date_min']).dt.days\nhist_trans_group['hist_purchase_date_average'] = hist_trans_group['hist_purchase_date_diff']/hist_trans_group['hist_card_id_size']\nhist_trans_group['hist_purchase_date_uptonow'] = (datetime.datetime.today() - hist_trans_group['hist_purchase_date_max']).dt.days\ntrain = train.merge(hist_trans_group,on='card_id',how='left')\ntest = test.merge(hist_trans_group,on='card_id',how='left')\ndel hist_trans_group;gc.collect();gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a88039f8d4c4ee2233df00d6f2f7316c0488a09e"},"cell_type":"code","source":"aggs = {}\nfor col in ['month','hour','weekofyear','dayofweek','year','subsector_id','merchant_id','merchant_category_id']:\n    aggs[col] = ['nunique']\naggs['purchase_amount'] = ['sum','max','min','mean','var']\naggs['installments'] = ['sum','max','min','mean','var']\naggs['purchase_date'] = ['max','min']\naggs['month_lag'] = ['max','min','mean','var']\naggs['month_diff'] = ['mean']\naggs['weekend'] = ['sum', 'mean']\naggs['category_1'] = ['sum', 'mean']\naggs['card_id'] = ['size']\n\nfor col in ['category_2','category_3']:\n    new_m_t[col+'_mean'] = new_m_t.groupby([col])['purchase_amount'].transform('mean')\n    aggs[col+'_mean'] = ['mean']\n    \nnew_columns = get_new_columns('new_hist',aggs)\nhist_trans_group = new_m_t.groupby('card_id').agg(aggs)\nhist_trans_group.columns = new_columns\nhist_trans_group.reset_index(drop=False,inplace=True)\nhist_trans_group['new_hist_purchase_date_diff'] = (hist_trans_group['new_hist_purchase_date_max'] - hist_trans_group['new_hist_purchase_date_min']).dt.days\nhist_trans_group['new_hist_purchase_date_average'] = hist_trans_group['new_hist_purchase_date_diff']/hist_trans_group['new_hist_card_id_size']\nhist_trans_group['new_hist_purchase_date_uptonow'] = (datetime.datetime.today() - hist_trans_group['new_hist_purchase_date_max']).dt.days\ntrain = train.merge(hist_trans_group,on='card_id',how='left')\ntest = test.merge(hist_trans_group,on='card_id',how='left')\ndel hist_trans_group;gc.collect();gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6cd1a5b360e35c2b8bfcad86e7eb725a128bd57e"},"cell_type":"code","source":"del his_t;gc.collect()\ndel new_m_t;gc.collect()\ntrain.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4012423085cd36b599d918165be45800a1951e8d"},"cell_type":"code","source":"train['outliers'] = 0\ntrain.loc[train['target'] < -30, 'outliers'] = 1\ntrain['outliers'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1dcc37d49cdc0081c9b994c040ffe120e86e082d"},"cell_type":"code","source":"for df in [train,test]:\n    df['first_active_month'] = pd.to_datetime(df['first_active_month'])\n    df['dayofweek'] = df['first_active_month'].dt.dayofweek\n    df['weekofyear'] = df['first_active_month'].dt.weekofyear\n    df['month'] = df['first_active_month'].dt.month\n    df['elapsed_time'] = (datetime.datetime.today() - df['first_active_month']).dt.days\n    df['hist_first_buy'] = (df['hist_purchase_date_min'] - df['first_active_month']).dt.days\n    df['new_hist_first_buy'] = (df['new_hist_purchase_date_min'] - df['first_active_month']).dt.days\n    for f in ['hist_purchase_date_max','hist_purchase_date_min','new_hist_purchase_date_max',\\\n                     'new_hist_purchase_date_min']:\n        df[f] = df[f].astype(np.int64) * 1e-9\n    df['card_id_total'] = df['new_hist_card_id_size']+df['hist_card_id_size']\n    df['purchase_amount_total'] = df['new_hist_purchase_amount_sum']+df['hist_purchase_amount_sum']\n\nfor f in ['feature_1','feature_2','feature_3']:\n    order_label = train.groupby([f])['outliers'].mean()\n    train[f] = train[f].map(order_label)\n    test[f] = test[f].map(order_label)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68ab9b556078338b030a73ff4ea69eb151ed28f6"},"cell_type":"code","source":"train_columns = [c for c in train.columns if c not in ['card_id', 'first_active_month','target','outliers']]\ntarget = train['target']\ndel train['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2477b18ce88d1c60471f994d36fba211b08860f8"},"cell_type":"code","source":"#train_x=train[train_columns]\n#test_x=test[train_columns]\n\n#train[train_columns].to_csv('train_3.csv', index=False)\n#test[train_columns].to_csv('test_3.csv', index=False)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c26be329dd27c297171ca09353a5741329588fec"},"cell_type":"code","source":"param = {'num_leaves': 31,\n         'min_data_in_leaf': 30, \n         'objective':'regression',\n         'max_depth': -1,\n         'learning_rate': 0.01,\n         \"min_child_samples\": 20,\n         \"boosting\": \"gbdt\",\n         \"feature_fraction\": 0.9,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.9 ,\n         \"bagging_seed\": 11,\n         \"metric\": 'rmse',\n         \"lambda_l1\": 0.1,\n         \"verbosity\": -1,\n         \"nthread\": 4,\n         \"random_state\": 4590}\n\noof_lgb_3 = np.zeros(len(train))\npredictions_lgb_3 = np.zeros(len(test))\nstart = time.time()\n\nfolds = StratifiedKFold(n_splits=5, shuffle=True, random_state=15)\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train,train['outliers'].values)):    \n    print(\"fold n°{}\".format(fold_))\n    trn_data = lgb.Dataset(train.iloc[trn_idx][train_columns], label=target.iloc[trn_idx])\n    val_data = lgb.Dataset(train.iloc[val_idx][train_columns], label=target.iloc[val_idx])\n\n    num_round = 10000\n    clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=100, early_stopping_rounds = 200)\n    oof_lgb_3[val_idx] = clf.predict(train.iloc[val_idx][train_columns], num_iteration=clf.best_iteration)\n    \n    predictions_lgb_3 += clf.predict(test[train_columns], num_iteration=clf.best_iteration) / folds.n_splits\n\nnp.save('oof_lgb_3', oof_lgb_3)\nnp.save('predictions_lgb_3', predictions_lgb_3)\nprint(\"CV score: {:<8.5f}\".format(mean_squared_error(oof_lgb_3, target)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ac81c8466959ef8a3478abd4add86ff0b4fa887"},"cell_type":"code","source":"xgb_params = {'eta': 0.005, 'max_depth': 10, 'subsample': 0.8, 'colsample_bytree': 0.8, \n          'objective': 'reg:linear', 'eval_metric': 'rmse', 'silent': True}\n\noof_xgb_3 = np.zeros(len(train))\npredictions_xgb_3 = np.zeros(len(test))\n\nfolds = StratifiedKFold(n_splits=5, shuffle=True, random_state=15)\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train,train['outliers'].values)):    \n    print('-')\n    print(\"Fold {}\".format(fold_ + 1))\n    trn_data = xgb.DMatrix(data=train.iloc[trn_idx][train_columns], label=target.iloc[trn_idx])\n    val_data = xgb.DMatrix(data=train.iloc[val_idx][train_columns], label=target.iloc[val_idx])\n    watchlist = [(trn_data, 'train'), (val_data, 'valid')]\n    print(\"xgb \" + str(fold_) + \"-\" * 50)\n    num_round = 10000\n    xgb_model = xgb.train(xgb_params, trn_data, num_round, watchlist, early_stopping_rounds=50, verbose_eval=1000)\n    oof_xgb_3[val_idx] = xgb_model.predict(xgb.DMatrix(train.iloc[val_idx][train_columns]), ntree_limit=xgb_model.best_ntree_limit+50)\n\n    predictions_xgb_3 += xgb_model.predict(xgb.DMatrix(test[train_columns]), ntree_limit=xgb_model.best_ntree_limit+50) / folds.n_splits\n    \nnp.save('oof_xgb_3', oof_xgb_3)\nnp.save('predictions_xgb_3', predictions_xgb_3)\nnp.sqrt(mean_squared_error(target.values, oof_xgb_3))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bffbe7e045f02bb6448a48dbaa486572ceb38646"},"cell_type":"code","source":"train_stack = np.vstack([oof_lgb_3, oof_xgb_3]).transpose()\ntest_stack = np.vstack([predictions_lgb_3, predictions_xgb_3]).transpose()\n\nfolds = StratifiedKFold(n_splits=5, shuffle=True, random_state=15)\noof = np.zeros(train_stack.shape[0])\npredictions = np.zeros(test_stack.shape[0])\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train_stack, train['outliers'].values)):\n    print(\"fold n°{}\".format(fold_))\n    trn_data, trn_y = train_stack[trn_idx], target.iloc[trn_idx].values\n    val_data, val_y = train_stack[val_idx], target.iloc[val_idx].values\n\n    clf = Ridge(alpha=1)\n    clf.fit(trn_data, trn_y)\n    \n    oof[val_idx] = clf.predict(val_data)\n    predictions += clf.predict(test_stack) / folds.n_splits\n\n\nnp.sqrt(mean_squared_error(target.values, oof))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e4b13629845e9ac1f621a388eed5128e6e01251"},"cell_type":"code","source":"sample_submission = pd.read_csv('../input/sample_submission.csv')\nsample_submission['target'] = predictions\nsample_submission.to_csv('stacker.csv', index=False)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}