{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nimport datetime\nwarnings.filterwarnings(\"ignore\")\nimport gc\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a7aa9c08839a1e263150b46f7b16769ef920b21a"},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20dc27176762c6b6ead4e64d3a8a6f6b838b036f"},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\", parse_dates=[\"first_active_month\"])\ntest = pd.read_csv(\"../input/test.csv\", parse_dates=[\"first_active_month\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2c7b00557e75be4f4e443db57f86a3340576c66"},"cell_type":"code","source":"new_m_trans=reduce_mem_usage(pd.read_csv(\"../input/new_merchant_transactions.csv\"))\nhist_trans=reduce_mem_usage(pd.read_csv(\"../input/historical_transactions.csv\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6057be09ef756086d4e3e6507171184cee378790"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dfc10ec144f680d52680bea73a74116755a919a6"},"cell_type":"code","source":"new_m_trans.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"722e706ddcd227c795c42e0c5f42f031f7e5783c"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"072a5630fae4d8abc21682cf5fff21db12c818b5"},"cell_type":"code","source":"plt.figure(figsize=(12,8))\nsns.distplot(train[\"target\"].values,kde=True)\nplt.xlabel(\"score\")\nplt.title(\"histogram of Loyality score\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7c4a0d9239bfd0b877717da7acecf9d256fe308f"},"cell_type":"markdown","source":"It shows clealy the data is not skewd and shows few outliers far away from the normal distribution. Its go and see what percentage of target is an outlier."},{"metadata":{"trusted":true,"_uuid":"42b38fff1f461491fdc8c1e52a638b24346441e8"},"cell_type":"code","source":"train[train[\"target\"]<-30].shape[0]/train.shape[0] * 100","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"71a94a9c390618beb3529f522bd02624e3c48881"},"cell_type":"markdown","source":"It shows around 1.09 percent of data is far away from the normal distribution. May be it will play an role in modeling. now lets hgo and look at the box plots."},{"metadata":{"trusted":true,"_uuid":"461524ad9be5aa963b89e514dd8bd37718c49509"},"cell_type":"code","source":"plt.figure(figsize=(10,8))\nimport seaborn as sns \nsns.boxplot(train[\"target\"].values)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"016625965e4a39e76b0f295dc758dd4fcaff37bc"},"cell_type":"markdown","source":"Lets look the violin plot so that we an get a good view of the outliers."},{"metadata":{"trusted":true,"_uuid":"057c2d6a097be5b032c4989882119aa2792dfa4e"},"cell_type":"code","source":"plt.figure(figsize=(10,8))\nsns.set_style(\"whitegrid\")\nsns.violinplot(train[\"target\"].values)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cb60dfeca6fb628dc51647ddae71f4e8083f3613"},"cell_type":"markdown","source":"we can see clearly a small bump after -30 which shows clearly the outliers. Lets look the feature1,2 & 3."},{"metadata":{"trusted":true,"_uuid":"5b3539163f04adfd9e2a7050325cd080d1791432"},"cell_type":"code","source":"f, (ax1,ax2,ax3) = plt.subplots(ncols=3, figsize=(17, 6), sharex=True)\nsns.despine(left=True)\n\nsns.distplot(train[\"feature_1\"].values,kde=False,ax=ax1)\nsns.distplot(train[\"feature_2\"].values,kde=False,ax=ax2)\nsns.distplot(train[\"feature_3\"].values,kde=False,ax=ax3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"933a2c55853e9ef79f251cd5ce18d8c2dabe8b60"},"cell_type":"markdown","source":"\n\nThis shows that the Feature_1,2,3 are the discreate values of (0,1,2,3,4,5,) may be these are the categorical values."},{"metadata":{"trusted":true,"_uuid":"79701b69f0a0e744e8d485da6183a413d0d2525c"},"cell_type":"code","source":"new_m_trans.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3285975548f98c9aa71f876f75b0268972103b62"},"cell_type":"code","source":"new_m_trans.category_3.value_counts(),new_m_trans.category_2.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96f29a7ce483534d7aeba5bb98998e8854649135"},"cell_type":"code","source":"dic=pd.read_excel(\"../input/Data_Dictionary.xlsx\")\ndic","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93a1d7768431fcd1054f8491e03d9d1f12c1a4e9"},"cell_type":"code","source":"merchants=pd.read_csv(\"../input/merchants.csv\")\nmerchants.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a49d370dd7da1d0b50245094876136b30533105d"},"cell_type":"code","source":"new_m_trans.installments.value_counts()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be204ac392531e4a42673109eeabc99b2f41116b"},"cell_type":"code","source":"train=pd.get_dummies(train,columns=[\"feature_1\",\"feature_2\"])\ntest=pd.get_dummies(test,columns=[\"feature_1\",\"feature_2\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1a3cee9567c084d9defa202ac25e7f4d7acb722"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f7d21e9f9f6cb54a21d0e908f0ca34fe1c2b74e"},"cell_type":"code","source":"for df in [new_m_trans,hist_trans]:\n    df[\"category_2\"].fillna(df[\"category_2\"].mode(),inplace=True,)\n    df[\"category_3\"].fillna(\"A\",inplace=True)\n    df['merchant_id'].fillna('M_ID_00a6ca8a8a',inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f0424f777556be83a27db3f9092e07e4ff6ae36"},"cell_type":"code","source":"hist_trans.installments.replace(999,0,inplace=True)\nhist_trans.installments.replace(-1,0,inplace=True)\nnew_m_trans.installments.replace(999,0,inplace=True)\nnew_m_trans.installments.replace(-1,0,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a7f8725eb0d2cd25f7ba85972ce2075f3655742"},"cell_type":"code","source":"for df in [hist_trans,new_m_trans]:\n    df[\"purchase_date\"]=pd.to_datetime(df[\"purchase_date\"])\n    df[\"year\"]=df[\"purchase_date\"].dt.year\n    df[\"weekofyear\"]=df[\"purchase_date\"].dt.weekofyear\n    df[\"month\"]=df[\"purchase_date\"].dt.month\n    df[\"dayofweek\"]=df[\"purchase_date\"].dt.dayofweek\n    df['weekend'] = (df.purchase_date.dt.weekday >=5).astype(int)\n    df[\"hour\"]=df[\"purchase_date\"].dt.hour\n    df[\"authorized_flag\"]=df[\"authorized_flag\"].map({\"Y\":1,\"N\":0})\n    df[\"category_1\"]=df[\"category_1\"].map({\"Y\":1,\"N\":0})\n    df[\"month_diff\"]=((datetime.datetime.today() - df['purchase_date']).dt.days)//30\n    df[\"month_diff\"]+= df['month_lag']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00e073776b7a344efe66b957981b796107317855"},"cell_type":"code","source":"hist_trans = pd.get_dummies(hist_trans, columns=['category_2', 'category_3'])\nnew_m_trans=pd.get_dummies(new_m_trans, columns=[\"category_2\",\"category_3\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2dc67c04bffefc4fbb4cc1da33ed51995eb9d88f"},"cell_type":"code","source":"hist_trans.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"22ac68fed3da64d214f61adaecb29d2dc828e97a"},"cell_type":"code","source":"def get_new_columns(name,aggs):\n    return [name + '_' + k + '_' + agg for k in aggs.keys() for agg in aggs[k]]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3177767b460aedbbe1b0bbacb291cd246d4f6e17"},"cell_type":"code","source":"aggs = {}\nfor col in ['month','hour','weekofyear','dayofweek','year','subsector_id','merchant_id','merchant_category_id','state_id']:\n    aggs[col] = ['nunique']\n\naggs['purchase_amount'] = ['sum','max','min','mean','var']\naggs['installments'] = ['sum','max','min','mean','var']\naggs['purchase_date'] = ['max','min']\naggs['month_lag'] = ['max','min','mean','var']\naggs['month_diff'] = [\"min\",\"max\",'mean']\naggs['authorized_flag'] = ['sum', 'mean']\naggs['weekend'] = ['sum', 'mean']\naggs['category_1'] = ['sum', 'mean']\naggs['card_id'] = ['size']\naggs['category_1']= ['sum','mean']\naggs['category_2_1.0']= ['mean','sum']\naggs['category_2_2.0']= ['mean','sum']\naggs['category_2_3.0']= ['mean','sum']\naggs['category_2_4.0']=['mean','sum']\naggs['category_2_5.0']= ['mean','sum']\naggs['category_3_A']= ['mean','sum']\naggs['category_3_B']= ['mean','sum']\naggs['category_3_C']= ['mean','sum']\n\n\n\nnew_columns = get_new_columns('hist',aggs)\nhist_trans_group = hist_trans.groupby('card_id').agg(aggs)\nhist_trans_group.columns = new_columns\nhist_trans_group.reset_index(drop=False,inplace=True)\nhist_trans_group['hist_purchase_date_diff'] = (hist_trans_group['hist_purchase_date_max'] - hist_trans_group['hist_purchase_date_min']).dt.days\nhist_trans_group['hist_purchase_date_average'] = hist_trans_group['hist_purchase_date_diff']/hist_trans_group['hist_card_id_size']\nhist_trans_group['hist_purchase_date_uptonow'] = (datetime.datetime.today() - hist_trans_group['hist_purchase_date_max']).dt.days\ntrain = train.merge(hist_trans_group,on='card_id',how='left')\ntest = test.merge(hist_trans_group,on='card_id',how='left')\ndel hist_trans_group;gc.collect();gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06726e8281815ff1dbf3d15cd81ae15cd86aed25"},"cell_type":"code","source":"aggs = {}\nfor col in ['month','hour','weekofyear','dayofweek','year','subsector_id','merchant_id','merchant_category_id','state_id']:\n    aggs[col] = ['nunique']\naggs['purchase_amount'] = ['sum','max','min','mean','var']\naggs['installments'] = ['sum','max','min','mean','var']\naggs['purchase_date'] = ['max','min']\naggs['month_lag'] = ['max','min','mean','var']\naggs['month_diff'] = [\"min\",\"max\",'mean']\naggs['authorized_flag'] = ['sum', 'mean']\naggs['weekend'] = ['sum', 'mean']\naggs['category_1'] = ['sum', 'mean']\naggs['card_id'] = ['size']\naggs['category_1']= ['sum','mean']\naggs['category_2_1.0']= ['mean','sum']\naggs['category_2_2.0']= ['mean','sum']\naggs['category_2_3.0']= ['mean','sum']\naggs['category_2_4.0']=['mean','sum']\naggs['category_2_5.0']= ['mean','sum']\naggs['category_3_A']= ['mean','sum']\naggs['category_3_B']= ['mean','sum']\naggs['category_3_C']= ['mean','sum']\n\n\n    \nnew_columns = get_new_columns('new_hist',aggs)\nhist_trans_group = new_m_trans.groupby('card_id').agg(aggs)\nhist_trans_group.columns = new_columns\nhist_trans_group.reset_index(drop=False,inplace=True)\nhist_trans_group['new_hist_purchase_date_diff'] = (hist_trans_group['new_hist_purchase_date_max'] - hist_trans_group['new_hist_purchase_date_min']).dt.days\nhist_trans_group['new_hist_purchase_date_average'] = hist_trans_group['new_hist_purchase_date_diff']/hist_trans_group['new_hist_card_id_size']\nhist_trans_group['new_hist_purchase_date_uptonow'] = (datetime.datetime.today() - hist_trans_group['new_hist_purchase_date_max']).dt.days\ntrain = train.merge(hist_trans_group,on='card_id',how='left')\ntest = test.merge(hist_trans_group,on='card_id',how='left')\ndel hist_trans_group;gc.collect();gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"5741bea78fb7da74d83ea959546c74d95406fa43"},"cell_type":"code","source":"\ndel hist_trans;gc.collect()\ndel new_m_trans;gc.collect()\ntrain.head(5)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df5e99b4058f50be7bbb9491209a9b84421c4eb2"},"cell_type":"code","source":"train['outliers'] = 0\ntrain.loc[train['target'] < -30, 'outliers'] = 1\ntrain['outliers'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c3abe27475ea856389e1e25841d14e918792c4f"},"cell_type":"code","source":"\nfor df in [train,test]:\n    df['first_active_month'] = pd.to_datetime(df['first_active_month'])\n    df['dayofweek'] = df['first_active_month'].dt.dayofweek\n    df['weekofyear'] = df['first_active_month'].dt.weekofyear\n    df['month'] = df['first_active_month'].dt.month\n    df['elapsed_time'] = (datetime.datetime.today() - df['first_active_month']).dt.days\n    df['hist_first_buy'] = (df['hist_purchase_date_min'] - df['first_active_month']).dt.days\n    df['new_hist_first_buy'] = (df['new_hist_purchase_date_min'] - df['first_active_month']).dt.days\n    for f in ['hist_purchase_date_max','hist_purchase_date_min','new_hist_purchase_date_max',\\\n                     'new_hist_purchase_date_min']:\n        df[f] = df[f].astype(np.int64) * 1e-9\n    df['card_id_total'] = df['new_hist_card_id_size']+df['hist_card_id_size']\n    df['purchase_amount_total'] = df['new_hist_purchase_amount_sum']+df['hist_purchase_amount_sum']\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8cad67c236b7002c32552ae5fa56e639d20fed87"},"cell_type":"code","source":"\nfor f in ['feature_1_1',\"feature_1_2\",\"feature_1_3\",\"feature_1_4\",\"feature_1_5\",'feature_2_1',\"feature_2_2\",\"feature_2_3\"]:\n    order_label = train.groupby([f])['outliers'].mean()\n    train[f] = train[f].map(order_label)\n    test[f] = test[f].map(order_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07b82602e34c9d48a5a21aa605d48af89e44ece6"},"cell_type":"code","source":"train_columns = [c for c in train.columns if c not in ['card_id', 'first_active_month','target','outliers']]\ntarget = train['target']\ndel train['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a04fc488d643eb6a04bac8242fec1e4923a02999"},"cell_type":"code","source":"\nimport lightgbm as lgb\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import KFold\nparam = {'num_leaves': 31,\n         'min_data_in_leaf': 30, \n         'objective':'regression',\n         'max_depth': -1,\n         'learning_rate': 0.01,\n         \"min_child_samples\": 20,\n         \"boosting\": \"gbdt\",\n         \"feature_fraction\": 0.9,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.9 ,\n         \"bagging_seed\": 11,\n         \"metric\": 'rmse',\n         \"lambda_l1\": 0.1,\n         \"verbosity\": -1,\n         \"nthread\": 4,\n         \"random_state\": 4590}\n\noof = np.zeros(len(train))\npredictions = np.zeros(len(test))\n#start = time.time()\n\nfolds = StratifiedKFold(n_splits=5, shuffle=True, random_state=15)\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train,train['outliers'].values)):    \n    print(\"fold n°{}\".format(fold_))\n    trn_data = lgb.Dataset(train.iloc[trn_idx][train_columns], label=target.iloc[trn_idx])\n    val_data = lgb.Dataset(train.iloc[val_idx][train_columns], label=target.iloc[val_idx])\n\n    num_round = 10000\n    clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=100, early_stopping_rounds = 200)\n    oof[val_idx] = clf.predict(train.iloc[val_idx][train_columns], num_iteration=clf.best_iteration)\n    \n    predictions += clf.predict(test[train_columns], num_iteration=clf.best_iteration) / folds.n_splits\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec8fe607f0479c24b74b4dd9b818d16628d4fab7"},"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nval_score = np.sqrt(mean_squared_error(target, oof))\nval_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be053da1d06ef23a6da0420469ceedb51d45ef29"},"cell_type":"code","source":"sample_submission = pd.read_csv('../input/sample_submission.csv')\nsample_submission['target'] = predictions\nsample_submission.to_csv('predict.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"747551199ff3e3e6f12503e583c9e5ae723c25af"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}