{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\"\"\"\n1/16/2019\nWhat has changed?\n1) Being NN, converted  categorical features through one-hot-encoder (was Lable Encoded earlier). This should make job easier for NN.\n\n##Attribution##\nThanks to Elo World, SRK and multiple other kernels. \nI have mixed data pre processing and feature engineering and tried ReLU with one hidden layer. \nI think we need to experiemnet with architecture and other hyperparameters.\n\"\"\"\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nfrom sklearn.linear_model import Ridge\nimport time\nfrom sklearn import preprocessing\nimport warnings\nimport datetime\nwarnings.filterwarnings(\"ignore\")\nimport gc\nfrom tqdm import tqdm\nfrom scipy.stats import describe\n%matplotlib inline\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import mean_squared_error\nimport xgboost as xgb\n# Any results you write to the current directory are saved as output.\nPATH = \"../input/\"\nprint(\"Path is:\",PATH)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9505afdb122250864abd652dc9d3891d45f2ed74"},"cell_type":"code","source":"#############################################################################################\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    return df\n################################################################################################\ntrain = pd.read_csv(PATH+'train.csv')\ntest = pd.read_csv(PATH+'test.csv')\nhist_trans = pd.read_csv(PATH+'historical_transactions.csv')\nnew_merchant_trans = pd.read_csv(PATH+'new_merchant_transactions.csv')\n\ntrain = reduce_mem_usage(train)\ntest = reduce_mem_usage(test)\nhist_trans = reduce_mem_usage(hist_trans)\nnew_merchant_trans = reduce_mem_usage(new_merchant_trans)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7acf5bceec4eed76a27b0be4b85ee6c246607ec1"},"cell_type":"code","source":"train = pd.get_dummies(train, columns= ['feature_1','feature_2','feature_3'], dummy_na= True)\ntest = pd.get_dummies(test, columns= ['feature_1','feature_2','feature_3'], dummy_na= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"15edc56dc606b1370a2a363252cf4786e674a053"},"cell_type":"code","source":"train.head(5)\nlen(train.columns)\ntrain.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2264519751ab6bcc708c5da0bfe0a9810d0843f8"},"cell_type":"code","source":"hist_trans = pd.get_dummies(hist_trans, columns= ['category_1','category_2','category_3'], dummy_na= True)\nnew_merchant_trans = pd.get_dummies(new_merchant_trans, columns= ['category_1','category_2','category_3'], dummy_na= True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"931259d0374e75fa461f4361835a1123fbaa723c","scrolled":true},"cell_type":"code","source":"new_merchant_trans.head(5)\nlen(new_merchant_trans.columns)\nnew_merchant_trans.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"706d6395e89eac2d6d1adc80f4cdb7356e587e5c"},"cell_type":"code","source":"\n\n############################################################################################    \n\nfor df in [hist_trans,new_merchant_trans]:\n    #df['category_2'].fillna(1.0,inplace=True)\n    #df['category_3'].fillna('A',inplace=True)\n    df['merchant_id'].fillna('M_ID_00a6ca8a8a',inplace=True)\n############################################################################################    \n    \ndef get_new_columns(name,aggs):\n    return [name + '_' + k + '_' + agg for k in aggs.keys() for agg in aggs[k]]\n############################################################################################    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fca62c3e5c7c11a266d17495430bff62fbe8280c"},"cell_type":"code","source":"\nfor df in [hist_trans,new_merchant_trans]:\n    df['purchase_date'] = pd.to_datetime(df['purchase_date'])\n    df['year'] = df['purchase_date'].dt.year\n    df['weekofyear'] = df['purchase_date'].dt.weekofyear\n    df['month'] = df['purchase_date'].dt.month\n    df['dayofweek'] = df['purchase_date'].dt.dayofweek\n    ###\n    #df['daysinmonth'] = df['purchase_date'].dt.daysinmonth\n    df['day'] = df['purchase_date'].dt.day\n    df['dayposinmon']=df['day']/(df['purchase_date'].dt.daysinmonth)\n    \n    \n    df['weekend'] = (df.purchase_date.dt.weekday >=5).astype(int)\n    df['hour'] = df['purchase_date'].dt.hour\n    df['authorized_flag'] = df['authorized_flag'].map({'Y':1, 'N':0})\n    #Handled by one_hot_encoder\n    #df['category_1'] = df['category_1'].map({'Y':1, 'N':0}) \n    #https://www.kaggle.com/c/elo-merchant-category-recommendation/discussion/73244\n    df['month_diff'] = ((datetime.datetime.today() - df['purchase_date']).dt.days)//30\n    df['month_diff'] += df['month_lag']\n    \n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2e8d961ada8d610cd232a9e7b864c9120bf9d8d"},"cell_type":"code","source":"hist_trans.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"950cc7c3e31c55273ce640f5840779667bd3e72a"},"cell_type":"code","source":"############################################################################################    \naggs = {}\nfor col in ['month','hour','weekofyear','dayofweek','year','subsector_id','merchant_id','merchant_category_id']:\n    aggs[col] = ['nunique']\n\naggs['purchase_amount'] = ['sum','max','min','mean','var']\naggs['installments'] = ['sum','max','min','mean','var']\naggs['purchase_date'] = ['max','min']\naggs['month_lag'] = ['max','min','mean','var']\naggs['month_diff'] = ['mean']\naggs['authorized_flag'] = ['sum', 'mean']\naggs['weekend'] = ['sum', 'mean']\n#aggs['category_1'] = ['sum', 'mean']\n#aggs['category_1_N','category_1_Y','category_1_nan'] = ['sum', 'mean']\naggs['card_id'] = ['size']\n\"\"\"\nfor col in ['category_2_1.0', 'category_2_2.0','category_2_3.0', 'category_2_4.0', 'category_2_5.0', 'category_2_nan','category_3_A', 'category_3_B', 'category_3_C', 'category_3_nan']:\n    hist_trans[col+'_mean'] = hist_trans.groupby([col])['purchase_amount'].transform('mean')\n    aggs[col+'_mean'] = ['mean']    \n\"\"\"\nnew_columns = get_new_columns('hist',aggs)\nhist_trans_group = hist_trans.groupby('card_id').agg(aggs)\nhist_trans_group.columns = new_columns\nhist_trans_group.reset_index(drop=False,inplace=True)\nhist_trans_group['hist_purchase_date_diff'] = (hist_trans_group['hist_purchase_date_max'] - hist_trans_group['hist_purchase_date_min']).dt.days\nhist_trans_group['hist_purchase_date_average'] = hist_trans_group['hist_purchase_date_diff']/hist_trans_group['hist_card_id_size']\nhist_trans_group['hist_purchase_date_uptonow'] = (datetime.datetime.today() - hist_trans_group['hist_purchase_date_max']).dt.days\ntrain = train.merge(hist_trans_group,on='card_id',how='left')\ntest = test.merge(hist_trans_group,on='card_id',how='left')\ndel hist_trans_group;gc.collect();gc.collect()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e10ce274cb2aeb2bda282f1f9951665528cef6e"},"cell_type":"code","source":"############################################################################################    \naggs = {}\nfor col in ['month','hour','weekofyear','dayofweek','year','subsector_id','merchant_id','merchant_category_id']:\n    aggs[col] = ['nunique']\naggs['purchase_amount'] = ['sum','max','min','mean','var']\naggs['installments'] = ['sum','max','min','mean','var']\naggs['purchase_date'] = ['max','min']\naggs['month_lag'] = ['max','min','mean','var']\naggs['month_diff'] = ['mean']\naggs['weekend'] = ['sum', 'mean']\n#Handled by one_hot_encoder\n#aggs['category_1'] = ['sum', 'mean']\n#aggs['category_1_N','category_1_Y','category_1_nan'] = ['sum', 'mean']\naggs['card_id'] = ['size']\n\"\"\"\nfor col in ['category_2_1.0', 'category_2_2.0','category_2_3.0', 'category_2_4.0', 'category_2_5.0', 'category_2_nan','category_3_A', 'category_3_B', 'category_3_C', 'category_3_nan']:\n    new_merchant_trans[col+'_mean'] = new_merchant_trans.groupby([col])['purchase_amount'].transform('mean')\n    aggs[col+'_mean'] = ['mean']\n\"\"\"  \nnew_columns = get_new_columns('new_hist',aggs)\nhist_trans_group = new_merchant_trans.groupby('card_id').agg(aggs)\nhist_trans_group.columns = new_columns\nhist_trans_group.reset_index(drop=False,inplace=True)\nhist_trans_group['new_hist_purchase_date_diff'] = (hist_trans_group['new_hist_purchase_date_max'] - hist_trans_group['new_hist_purchase_date_min']).dt.days\nhist_trans_group['new_hist_purchase_date_average'] = hist_trans_group['new_hist_purchase_date_diff']/hist_trans_group['new_hist_card_id_size']\nhist_trans_group['new_hist_purchase_date_uptonow'] = (datetime.datetime.today() - hist_trans_group['new_hist_purchase_date_max']).dt.days\ntrain = train.merge(hist_trans_group,on='card_id',how='left')\ntest = test.merge(hist_trans_group,on='card_id',how='left')\ndel hist_trans_group;gc.collect();gc.collect()\n############################################################################################    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8919918f08d43679fdf9b39ead632b95d9210571"},"cell_type":"code","source":"\ndel hist_trans;gc.collect()\ndel new_merchant_trans;gc.collect()\ntrain.head(5)\n\ntrain['outliers'] = 0\ntrain.loc[train['target'] < -30, 'outliers'] = 1\ntrain['outliers'].value_counts()\n############################################################################################    \n\n\nfor df in [train,test]:\n    df['first_active_month'] = pd.to_datetime(df['first_active_month'])\n    df['dayofweek'] = df['first_active_month'].dt.dayofweek\n    df['weekofyear'] = df['first_active_month'].dt.weekofyear\n    ###\n    #df['daysinmonth'] = df['first_active_month'].dt.daysinmonth\n    df['day'] = df['first_active_month'].dt.day\n    df['dayposinmon']=df['day']/(df['first_active_month'].dt.daysinmonth)\n    \n    df['month'] = df['first_active_month'].dt.month\n    df['elapsed_time'] = (datetime.datetime.today() - df['first_active_month']).dt.days\n    df['hist_first_buy'] = (df['hist_purchase_date_min'] - df['first_active_month']).dt.days\n    df['new_hist_first_buy'] = (df['new_hist_purchase_date_min'] - df['first_active_month']).dt.days\n    for f in ['hist_purchase_date_max','hist_purchase_date_min','new_hist_purchase_date_max',\\\n                     'new_hist_purchase_date_min']:\n        df[f] = df[f].astype(np.int64) * 1e-9\n    df['card_id_total'] = df['new_hist_card_id_size']+df['hist_card_id_size']  \n        \n    df['purchase_amount_total'] = df['new_hist_purchase_amount_sum']+df['hist_purchase_amount_sum']\n\nfor f in ['feature_1_1.0','feature_1_2.0', 'feature_1_3.0', 'feature_1_4.0', 'feature_1_5.0','feature_1_nan', 'feature_2_1.0', 'feature_2_2.0', 'feature_2_3.0','feature_2_nan', 'feature_3_0.0', 'feature_3_1.0', 'feature_3_nan']:\n        order_label = train.groupby([f])['outliers'].mean()\n        train[f] = train[f].map(order_label)\n        test[f] = test[f].map(order_label)\n  \ntrain_columns = [c for c in train.columns if c not in ['card_id', 'first_active_month','target','outliers']]\ntarget = train['target']\ndel train['target']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de7cbefdf085c7cce32dae0490c8bf58d4e879f7"},"cell_type":"code","source":"###############################################################ANN World ########################################\n\nfrom keras.layers import Input,Dropout,BatchNormalization,Activation,Add\nfrom sklearn.preprocessing import MinMaxScaler\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout\nfrom keras.regularizers import l2\n\n#####Getting the same number of columns for Train, Test######\n\ny = target\n\ntrain_df = train[train_columns]\ntest_df = test[train_columns]\n\n#####Handling Missing Values#####     \n\nfor i in range(len(train_df.columns)):\n    train_df.iloc[:,i] = (train_df.iloc[:,i]).fillna(-1)\n\nfor i in range(len(test_df.columns)):\n    test_df.iloc[:,i] = (test_df.iloc[:,i]).fillna(-1)    \n    \n#####Encoding the Categorical Variables#####\n\"\"\"\nlbl = LabelEncoder()\n\nfor c in train.columns:\n    if train[c].dtype == 'object':\n        lbl.fit(list(train[c].values))\n        train[c] = lbl.transform(list(train[c].values))\n\nfor c in test.columns:\n    if test[c].dtype == 'object':\n        lbl.fit(list(test[c].values))\n        test[c] = lbl.transform(list(test[c].values))     \n        \nprint(\"Done with the Encoding\")        \n\"\"\"\n####Normalizing the values####\n\nmmScale = MinMaxScaler()\n\nn = train_df.shape[1]\nprint('n is:',n)\n\nx_train = mmScale.fit_transform(train_df)\nx_test = mmScale.transform(test_df)\n\n\nfrom keras import optimizers\nfrom keras import layers\nfrom keras.layers import LeakyReLU\n\n#####################Early Stopping ########################\nfrom keras.callbacks import EarlyStopping\nearly_stopping = EarlyStopping(monitor='val_loss', patience=3)\n#####Artificial Neural Networks Implementation#####\nprint(\"Starting Neural Network\")\n\nmodel_n = Sequential()\n#Want to use an expotential linear unit instead of the usual relu\n#model_n.add( Dense( n, activation='relu', input_shape=(n,) ) )\nmodel_n.add( Dense( n, activation='tanh', input_shape=(n,) ) )\nmodel_n.add(BatchNormalization())\nmodel_n.add( Dense( int(0.5*n), activation='relu' ) )\nmodel_n.add(LeakyReLU(alpha=.0011))\n\nmodel_n.add(Dropout(0.5))\n\n\"\"\"\nmodel_n.add( Dense( 512, activation='relu' ) )\nmodel_n.add(BatchNormalization())\nmodel_n.add(LeakyReLU(alpha=.001))\nmodel_n.add(Dropout(0.5))\n\nmodel_n.add( Dense( 256, activation='relu' ) )\nmodel_n.add(BatchNormalization())\nmodel_n.add(LeakyReLU(alpha=.001))\nmodel_n.add(Dropout(0.5))\n\nmodel_n.add( Dense( 128, activation='relu' ) )\nmodel_n.add(BatchNormalization())\nmodel_n.add(LeakyReLU(alpha=.001))\nmodel_n.add(Dropout(0.5))\n\nmodel_n.add( Dense( 1024, activation='relu' ) )\nmodel_n.add(BatchNormalization())\nmodel_n.add(LeakyReLU(alpha=.001))\nmodel_n.add(Dropout(0.5))\n\nmodel_n.add( Dense( 99, activation='relu' ) )\nmodel_n.add(BatchNormalization())\nmodel_n.add(LeakyReLU(alpha=.001))\nmodel_n.add(Dropout(0.5))\n\nmodel_n.add( Dense( 64, activation='relu' ) )\nmodel_n.add(BatchNormalization())\nmodel_n.add(LeakyReLU(alpha=.001))\nmodel_n.add(Dropout(0.5))\n\nmodel_n.add( Dense( 19, activation='relu' ) )\nmodel_n.add(BatchNormalization())\nmodel_n.add(LeakyReLU(alpha=.001))\nmodel_n.add(Dropout(0.5))\n\"\"\"\nmodel_n.add(Dense(1, activation='linear'))\nmodel_n.compile(loss='mse', optimizer='Adadelta',  metrics=['mse'])\n        \n\nmodel_n.fit(x_train, y, epochs=25,verbose=1, batch_size=32,validation_split=0.2, callbacks=[early_stopping])\n\npredictions = model_n.predict(x_test)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9740e8d0ccaa6e47f7af03f5f830b113c63ad89"},"cell_type":"code","source":"sub_df = pd.DataFrame({\"card_id\":test[\"card_id\"]})\nsub_df[\"target\"] = pd.DataFrame(predictions)\nfrom datetime import datetime\nsub_df.to_csv('Kaggle_ELO_CNNStarter_{}.csv'.format(datetime.now().strftime('%Y%m%d_%H%M%S')), index=False,float_format='%.4f')\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f542d35b8fcd2d61379efd28c416e73ba328b7d8"},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.linear_model import LinearRegression\n\nclf = LinearRegression()\nclf.fit(x_train, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d9314f431fa46d095e013b5076992486128bf55"},"cell_type":"code","source":"pred_test_y = clf.predict(x_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"244000d9888905fd785a2402f0d7fa4ab1bd285d"},"cell_type":"code","source":"sub_df1 = pd.DataFrame({\"card_id\":test[\"card_id\"]})\nsub_df1[\"target\"] = pd.DataFrame(pred_test_y)\nfrom datetime import datetime\nsub_df1.to_csv('Kaggle_ELO_LRStarter_{}.csv'.format(datetime.now().strftime('%Y%m%d_%H%M%S')), index=False,float_format='%.4f')\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}