{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport lightgbm as lgb\nimport os\nimport time\nimport warnings\nimport gc\ngc.collect()\nimport os\nfrom six.moves import urllib\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport datetime\nwarnings.filterwarnings('ignore')\n%matplotlib inline\nplt.style.use('seaborn')\nfrom scipy.stats import norm, skew\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import KFold, StratifiedKFold, RepeatedKFold\nfrom sklearn.metrics import mean_squared_error\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df3a5e7662096bd28c344d33e7ab8be6dfa6326b"},"cell_type":"code","source":"#Reduce the memory usage - Thanks to Panchajanya Banerjee\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"#Reading data files\n\nhist_transactions = reduce_mem_usage (pd.read_csv('../input/historical_transactions.csv'))\n#transactions = reduce_mem_usage (pd.read_csv('../input/new_merchant_transactions.csv'))\ngc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"882a25174f2db917d45738038052634b1860bc48"},"cell_type":"code","source":"hist_transactions.head()\n#hist_transactions = old_transactions[old_transactions['card_id']=='C_ID_4e6213e9bc']\n#transactions = new_transactions[new_transactions['card_id']=='C_ID_4e6213e9bc']\n#del old_transactions\n#del new_transactions\ngc.collect","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"03526e6e15463f1a1607f46482c05a2dbf21b2aa"},"cell_type":"code","source":"#Replacing NAN values\nhist_transactions['category_3'] = hist_transactions['category_3'].replace(np.nan, 'A')\nhist_transactions['merchant_id'] = hist_transactions['merchant_id'].replace(np.nan, 'M_ID_00a6ca8a8a')\nhist_transactions['category_2'] = hist_transactions['category_2'].replace(np.nan, 1.0)\n#Converting  -- for now not running\nhist_transactions['authorized_flag'] = hist_transactions['authorized_flag'].map({'Y': 1, 'N': 0})\nhist_transactions['category_1'] = hist_transactions['category_1'].map({'Y': 1, 'N': 0})\nhist_transactions['category_3'] = hist_transactions['category_3'].map({'A':0, 'B':1, 'C':2})\n\nhist_transactions['purchase_date'] = pd.to_datetime(hist_transactions['purchase_date'])\nhist_transactions['year'] = hist_transactions['purchase_date'].dt.year\nhist_transactions['weekofyear'] = hist_transactions['purchase_date'].dt.weekofyear\nhist_transactions['month'] = hist_transactions['purchase_date'].dt.month\nhist_transactions['dayofweek'] = hist_transactions['purchase_date'].dt.dayofweek\nhist_transactions['weekend'] = (hist_transactions.purchase_date.dt.weekday >=5).astype(int)\nhist_transactions['hour'] = hist_transactions['purchase_date'].dt.hour \nhist_transactions['quarter'] = hist_transactions['purchase_date'].dt.quarter\nhist_transactions['is_month_start'] = hist_transactions['purchase_date'].dt.is_month_start\nhist_transactions['month_diff'] = ((datetime.datetime.today() - hist_transactions['purchase_date']).dt.days)//30\nhist_transactions['month_diff'] += hist_transactions['month_lag'] \n\nagg_values = {\n    'authorized_flag': ['nunique'],\n    'city_id' : ['nunique'], \n    'category_1' : ['nunique'],\n    'installments' : ['mean', 'max', 'min', 'var', 'std', 'sum'],\n    'category_3' : ['nunique'],\n    'merchant_category_id' : ['nunique'], \n     'month_lag' : ['mean', 'max', 'min', 'var', 'std', 'sum'],\n    'purchase_amount' : ['mean', 'max', 'min', 'var', 'std', 'sum'], \n     'category_2' : ['nunique'],\n    'state_id' : ['nunique'],\n    'subsector_id' : ['nunique'], \n    'year':  ['nunique'], \n    'weekofyear':  ['nunique'], \n    'month':  ['nunique'], \n    'dayofweek':  ['nunique'], \n    'weekend':  ['nunique'], \n     'hour':  ['nunique'], \n     'quarter':  ['nunique'], \n     'is_month_start':  ['nunique'], \n     'month_diff': ['mean', 'max', 'min', 'var', 'std', 'nunique'],\n    'purchase_date' : ['max','min'],\n    'card_id' :['count']\n}\n\nhist_aggregate = hist_transactions \\\n    .groupby('card_id') \\\n    .agg(agg_values) \\\n    .reset_index()\n\ndel hist_transactions\ngc.collect()\n\nhist_aggregate.columns = ['col_' + '_'.join(col).strip() \n                            for col in hist_aggregate.columns.values]\n\nhist_aggregate.rename(columns={'col_card_id_':'card_id'}, inplace=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4766adcf57b2717811233cf667a517fd839a71cd"},"cell_type":"code","source":"train  = reduce_mem_usage (pd.read_csv('../input/train.csv',parse_dates=[\"first_active_month\"]))\ntest = reduce_mem_usage (pd.read_csv('../input/test.csv',parse_dates=[\"first_active_month\"]))\n#Histogram to get the distribution and Outlier\ntrain['target'].plot(kind='hist') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1548b18eb58fbbf842556a24407282136852bb80"},"cell_type":"code","source":"# Remove the Outliers if any -- Adding a new column\ntrain['outliers'] = 0\ntrain.loc[train['target'] < -30, 'outliers'] = 1\ntrain['outliers'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef8809a48bc06d7f8c2bfa82f2d024624ff8b410"},"cell_type":"code","source":"# Now extract the month, year, day, weekday\ntrain[\"month\"] = train[\"first_active_month\"].dt.month\ntrain[\"year\"] = train[\"first_active_month\"].dt.year\ntrain['week'] = train[\"first_active_month\"].dt.weekofyear\ntrain['dayofweek'] = train['first_active_month'].dt.dayofweek\ntrain['days'] = (datetime.date(2018, 2, 1) - train['first_active_month'].dt.date).dt.days\ntrain['quarter'] = train['first_active_month'].dt.quarter\ntrain['is_month_start'] = train['first_active_month'].dt.is_month_start\n\n\ntest[\"month\"] = test[\"first_active_month\"].dt.month\ntest[\"year\"] = test[\"first_active_month\"].dt.year\ntest['week'] = test[\"first_active_month\"].dt.weekofyear\ntest['dayofweek'] = test['first_active_month'].dt.dayofweek\ntest['days'] = (datetime.date(2018, 2, 1) - test['first_active_month'].dt.date).dt.days\ntest['quarter'] = test['first_active_month'].dt.quarter\ntest['is_month_start'] = test['first_active_month'].dt.is_month_start","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f956a36b0aa122d8c3c790d1591f71299f5ea77c"},"cell_type":"code","source":"train = pd.merge(train, hist_aggregate, on='card_id',how='left')\ntest = pd.merge(test, hist_aggregate, on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ffdcb27a9bcb2182a213fcefbbc678b03f3305e"},"cell_type":"code","source":"transactions = reduce_mem_usage (pd.read_csv('../input/new_merchant_transactions.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20e2762e3ef2a716b99358fc7ecefc683ed2ba8d"},"cell_type":"code","source":"transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fbb50165cec576963e4f5619185cac043faaafd9"},"cell_type":"code","source":"#Replacing NAN values\ntransactions['category_3'] = transactions['category_3'].replace(np.nan, 'A')\ntransactions['merchant_id'] = transactions['merchant_id'].replace(np.nan, 'M_ID_00a6ca8a8a')\ntransactions['category_2'] = transactions['category_2'].replace(np.nan, 1.0)\n#Converting  -- for now not running\ntransactions['authorized_flag'] = transactions['authorized_flag'].map({'Y': 1, 'N': 0})\ntransactions['category_1'] = transactions['category_1'].map({'Y': 1, 'N': 0})\ntransactions['category_3'] = transactions['category_3'].map({'A':0, 'B':1, 'C':2})\n\ntransactions['purchase_date'] = pd.to_datetime(transactions['purchase_date'])\ntransactions['year'] = transactions['purchase_date'].dt.year\ntransactions['weekofyear'] = transactions['purchase_date'].dt.weekofyear\ntransactions['month'] = transactions['purchase_date'].dt.month\ntransactions['dayofweek'] = transactions['purchase_date'].dt.dayofweek\ntransactions['weekend'] = (transactions.purchase_date.dt.weekday >=5).astype(int)\ntransactions['hour'] = transactions['purchase_date'].dt.hour \ntransactions['quarter'] = transactions['purchase_date'].dt.quarter\ntransactions['is_month_start'] = transactions['purchase_date'].dt.is_month_start\ntransactions['month_diff'] = ((datetime.datetime.today() - transactions['purchase_date']).dt.days)//30\ntransactions['month_diff'] += transactions['month_lag'] \n\nagg_values = {\n    'authorized_flag': ['nunique'],\n    'city_id' : ['nunique'], \n    'category_1' : ['nunique'],\n    'installments' : ['mean', 'max', 'min', 'var', 'std', 'sum'],\n    'category_3' : ['nunique'],\n    'merchant_category_id' : ['nunique'], \n     'month_lag' : ['mean', 'max', 'min', 'var', 'std', 'sum'],\n    'purchase_amount' : ['mean', 'max', 'min', 'var', 'std', 'sum'], \n     'category_2' : ['nunique'],\n    'state_id' : ['nunique'],\n    'subsector_id' : ['nunique'], \n    'year':  ['nunique'], \n    'weekofyear':  ['nunique'], \n    'month':  ['nunique'], \n    'dayofweek':  ['nunique'], \n    'weekend':  ['nunique'], \n     'hour':  ['nunique'], \n     'quarter':  ['nunique'], \n     'is_month_start':  ['nunique'], \n     'month_diff': ['mean', 'max', 'min', 'var', 'std', 'nunique'],\n    'purchase_date' : ['max','min'],\n    'card_id' :['count']\n}\n\ntrans_aggregate = transactions \\\n    .groupby('card_id') \\\n    .agg(agg_values) \\\n    .reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa5de60e4704c87684c2e2ba67cbd0616a18b04a"},"cell_type":"code","source":"trans_aggregate.columns = ['trans_' + '_'.join(col).strip() \n                            for col in trans_aggregate.columns.values]\n\ntrans_aggregate.rename(columns={'trans_card_id_':'card_id'}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c099cd4ede8ec041571cb2eeeaab6963e8a7dfaf"},"cell_type":"code","source":"train = pd.merge(train, trans_aggregate, on='card_id',how='left')\ntest = pd.merge(test, trans_aggregate, on='card_id',how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"588c1b490541caf6999a5385c0a17c79793e08d0"},"cell_type":"code","source":"del trans_aggregate\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f42203fb4636e63703a565444317e87add10ee4"},"cell_type":"code","source":"FEATS_EXCLUDED= ['first_active_month', 'target', 'card_id', 'outliers',\n                  'hist_purchase_date_max', 'hist_purchase_date_min', 'hist_card_id_count',\n                  'trans_purchase_date_max', 'trans_purchase_date_min', 'trans_card_id_count','col_purchase_date_max_y',\n                'col_purchase_date_min_y', 'col_card_id_count_y','col_purchase_date_max_x','col_purchase_date_min_x',\n                'col_purchase_date_max','col_purchase_date_min']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bbb68d64bb5c6ec07563cf275b53dee9381fe64c"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2bf01c98507f0c591c6e5a28dd1bd52282289cd"},"cell_type":"code","source":"#Repeated Kfold yield better results than StratifiedKFold\n#folds = StratifiedKFold(n_splits= 10, shuffle=True, random_state=326)\nfolds = RepeatedKFold(n_splits=5, n_repeats=2, random_state=4950)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bcb1f8864eb5197e106b4808102cfdf8ae6c7c0e","scrolled":true},"cell_type":"code","source":"print(\"Starting LightGBM. Train shape: {}, test shape: {}\".format(train.shape, test.shape))\noof_preds = np.zeros(train.shape[0])\nsub_preds = np.zeros(test.shape[0])\nfeature_importance_df = pd.DataFrame()\nfeats = [f for f in train.columns if f not in FEATS_EXCLUDED]\n\nfor n_fold, (train_idx, valid_idx) in enumerate(folds.split(train[feats], train['outliers'])):\n#for n_fold, (train_idx, valid_idx) in enumerate(folds.split(train[feats])):\n        train_x, train_y = train[feats].iloc[train_idx], train['target'].iloc[train_idx]\n        valid_x, valid_y = train[feats].iloc[valid_idx], train['target'].iloc[valid_idx]\n\n        # set data structure\n        lgb_train = lgb.Dataset(train_x,\n                                label=train_y,\n                                free_raw_data=False)\n        lgb_test = lgb.Dataset(valid_x,\n                               label=valid_y,\n                               free_raw_data=False)\n\n       \n        params ={\n                'task': 'train',\n                'boosting': 'goss',\n                'objective': 'regression',\n                'metric': 'rmse',\n                'learning_rate': 0.01,\n                'subsample': 0.9855232997390695,\n                'max_depth': 7,\n                'top_rate': 0.9064148448434349,\n                'num_leaves': 63,\n                'min_child_weight': 41.9612869171337,\n                'other_rate': 0.0721768246018207,\n                'reg_alpha': 9.677537745007898,\n                'colsample_bytree': 0.5665320670155495,\n                'min_split_gain': 9.820197773625843,\n                'reg_lambda': 8.2532317400459,\n                'min_data_in_leaf': 21,\n                'verbose': -1,\n                'seed':int(2**n_fold),\n                'bagging_seed':int(2**n_fold),\n                'drop_seed':int(2**n_fold)\n                }\n\n        reg = lgb.train(\n                        params,\n                        lgb_train,\n                        valid_sets=[lgb_train, lgb_test],\n                        valid_names=['train', 'test'],\n                        num_boost_round=10000,\n                        early_stopping_rounds= 200,\n                        verbose_eval=100\n                        )\n\n        oof_preds[valid_idx] = reg.predict(valid_x, num_iteration=reg.best_iteration)\n        sub_preds += reg.predict(test[feats], num_iteration=reg.best_iteration) / (5*2)\n\n        fold_importance_df = pd.DataFrame()\n        fold_importance_df[\"feature\"] = feats\n        fold_importance_df[\"importance\"] = np.log1p(reg.feature_importance(importance_type='gain', iteration=reg.best_iteration))\n        fold_importance_df[\"fold\"] = n_fold + 1\n        feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0) \n        del reg, train_x, train_y, valid_x, valid_y\n        gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b4c8cd1afeeb1f92aa9eb7f7ea5b692dbb97e119"},"cell_type":"code","source":"submission_file_name= \"submission_v4.csv\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4dbb457f2ad3faae8012a44b792a63bcf9213f74"},"cell_type":"code","source":"def display_importances(feature_importance_df_):\n    cols = feature_importance_df_[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(by=\"importance\", ascending=False)[:40].index\n    best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]\n\n    plt.figure(figsize=(8, 10))\n    sns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\", ascending=False))\n    plt.title('LightGBM Features (avg over folds)')\n    plt.tight_layout()\n    plt.savefig('lgbm_importances.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4eac32fbfafa2e8345fd131026ed63d614feb91a"},"cell_type":"code","source":"display_importances(feature_importance_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4d2030850c947e3f9d75b3a00761c4fddf4a7de"},"cell_type":"code","source":"test.loc[:,'target'] = sub_preds\ntest = test.reset_index()\ntest[['card_id', 'target']].to_csv(submission_file_name, index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}