{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom pandas import DataFrame as df\n\n# train_data = pd.read_csv('../input/elo-merchant-category-recommendation/train.csv')\n# td_card = np.unique(train_data['card_id']) # 201917\n\n# ratio = 0.005\n# card_num = int(np.size(td_card, 0) * ratio)\n# sampled_card = np.random.choice(td_card, card_num)\n# del td_card\n\n# samp_c = df(sampled_card,columns=['card_id'])\n# del sampled_card\n\n# filter1 = train_data[\"card_id\"].isin(samp_c.card_id.tolist())\n# td = train_data[filter1]\n# del train_data, filter1\n\n# print(\"Reading historical_transactions...............\")\n# historical_data = pd.read_csv('../input/elo-merchant-category-recommendation/historical_transactions.csv')\n# filter2 = historical_data[\"card_id\"].isin(samp_c.card_id.tolist())\n# hd = historical_data[filter2]\n# del historical_data, filter2\n\n# print(\"Reading new_merchant_transactions...............\")\n# merchant_data = pd.read_csv('../input/elo-merchant-category-recommendation/new_merchant_transactions.csv')\n# filter3 = merchant_data[\"card_id\"].isin(samp_c.card_id.tolist())\n# md = merchant_data[filter3]\n# del merchant_data, filter3\n\n# frames = [hd, md]\n# hmd = pd.concat(frames)\n# print(hmd)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# td.to_csv(r'/kaggle/working/td005_1.csv')\n# hmd.to_csv(r'/kaggle/working/hmd005_1.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom pandas import DataFrame as df\nimport datetime\nimport lightgbm as lgb\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# train_data = pd.read_csv('../input/elo-dmbi-project/td005_1.csv')\n\nimport gc\n\nhist = pd.read_csv('../input/elo-merchant-category-recommendation/historical_transactions.csv', parse_dates=['purchase_date'])\nprint(hist.shape)\n\nnew_merch = pd.read_csv('../input/elo-merchant-category-recommendation/new_merchant_transactions.csv', parse_dates=['purchase_date'])\nprint(new_merch.shape)\n\nframes = [hist, new_merch]\ntransaction_data = pd.concat(frames)\nprint(transaction_data.shape)\ndel hist, new_merch\ngc.collect()\n\n\n# transaction_data = pd.read_csv('../input/elo-dmbi-project/hmd005_1.csv',parse_dates=['purchase_date'])\n\n\n#delete inconsistent data\nindexNames = transaction_data[ transaction_data['city_id'] == -1].index\ntransaction_data.drop(indexNames , inplace=True)\n\nindexNames = transaction_data[ transaction_data['category_2'].isna() ].index\ntransaction_data.drop(indexNames , inplace=True)\n\nindexNames = transaction_data[ transaction_data['installments'] == -1].index\ntransaction_data.drop(indexNames , inplace=True)\n\nindexNames = transaction_data[ transaction_data['installments'] == 999].index\ntransaction_data.drop(indexNames , inplace=True)\n\nindexNames = transaction_data[ transaction_data['purchase_amount'] > 1].index\ntransaction_data.drop(indexNames , inplace=True)\n\n# indexNames = transaction_data[ transaction_data['authorized_flag'] == \"N\"].index\n# transaction_data.drop(indexNames , inplace=True)\n\n# dropping ALL duplicte values \ntransaction_data.drop_duplicates(subset =['card_id','merchant_id','purchase_date' ], keep = 'first', inplace = True) \n\ntransaction_data['subsector_id_copy'] = transaction_data['subsector_id']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transaction_data  = transaction_data.fillna(0)\nprint(transaction_data.isna().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def binarize(df):\n    for col in ['authorized_flag', 'category_1']:\n        df[col] = df[col].map({'Y':1, 'N':0})\n    return df\n\ntransactions = binarize(transaction_data)\n\ntransactions = pd.get_dummies(transactions, columns=['category_2', 'category_3', 'subsector_id'])\n# transactions = pd.get_dummies(transactions, columns=['category_2', 'category_3'])\n\n\ntransactions['month_diff'] = ((datetime.datetime.today() - transactions['purchase_date']).dt.days)//30\ntransactions['month_diff'] += transactions['month_lag']\n\ntransactions[:5]\n\ndel transaction_data\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df):\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n\ntransactions = reduce_mem_usage(transactions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"merchants = pd.read_csv('../input/elo-merchant-category-recommendation/merchants.csv')\nmerchants[\"category_1\"] = merchants[\"category_1\"].map({'Y':1, 'N':0})\nmerchants[\"category_4\"] = merchants[\"category_4\"].map({'Y':1, 'N':0})\nmerchants = merchants.rename(columns={\"category_1\": \"mer_category_1\", \"category_2\": \"mer_category_2\",\"category_4\": \"mer_category_4\", \"city_id\":\"mer_city_id\"})\n\nprint(\"merchants file read \")\n\nmerchant_columns = [\"merchant_id\", \"mer_category_4\", \"mer_city_id\", \"merchant_group_id\" ]\n\nmerchants =merchants[merchant_columns]\n\n\ntransactions = pd.merge(transactions, merchants, on='merchant_id', how='left')\n\ntransactions[\"same_city\"]=np.where(transactions['city_id'] == transactions['mer_city_id'] , 1, 0)\ndel merchants","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# print(transactions.columns)\n# print(merchants.columns)\ntransactions[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transactions.to_csv(r'/kaggle/working/transactions.csv')\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def aggregate_transactions(data):\n    \n    data.loc[:, 'purchase_date'] = pd.DatetimeIndex(data['purchase_date']).\\\n                astype(np.int64) * 1e-9\n     \n    agg_func = {\n    'authorized_flag': ['mean'],\n    'category_1': ['mean'],\n    'category_2_1.0': ['mean'],\n    'category_2_2.0': ['mean'],\n    'category_2_3.0': ['mean'],\n    'category_2_4.0': ['mean'],\n    'category_2_5.0': ['mean'],\n    'category_3_A': ['mean'],\n    'category_3_B': ['mean'],\n    'category_3_C': ['mean'],\n    'merchant_id': ['nunique'],\n    'merchant_category_id': ['nunique'],\n    'merchant_group_id': ['nunique'],\n\n    'mer_category_4': ['mean'],\n    \n    'state_id': ['nunique'],\n    'city_id': ['nunique'],\n    'same_city': ['mean'],\n        \n#       'subsector_id': ['nunique'],  \n    'subsector_id_copy': ['nunique'],\n    'subsector_id_1':['mean'],\n    'subsector_id_2':['mean'],\n    'subsector_id_3':['mean'],\n    'subsector_id_4':['mean'],\n   'subsector_id_5':['mean'],\n    'subsector_id_7':['mean'],\n    'subsector_id_8':['mean'],\n    'subsector_id_9':['mean'],\n    'subsector_id_10':['mean'],\n    'subsector_id_11':['mean'],\n    'subsector_id_12':['mean'],\n    'subsector_id_13':['mean'],\n    'subsector_id_14':['mean'],\n    'subsector_id_15':['mean'],\n    'subsector_id_16':['mean'],\n    'subsector_id_17':['mean'],\n    'subsector_id_18':['mean'],\n    'subsector_id_19':['mean'],\n    'subsector_id_20':['mean'],\n    'subsector_id_21':['mean'],\n    'subsector_id_22':['mean'],\n    'subsector_id_23':['mean'],\n    'subsector_id_24':['mean'],\n    'subsector_id_25':['mean'],\n    'subsector_id_26':['mean'],\n    'subsector_id_27':['mean'],\n    'subsector_id_29':['mean'],\n    'subsector_id_30':['mean'],\n    'subsector_id_31':['mean'],\n    'subsector_id_32':['mean'],\n    'subsector_id_33':['mean'],\n    'subsector_id_34':['mean'],\n    'subsector_id_35':['mean'],\n    'subsector_id_36':['mean'],\n    'subsector_id_37':['mean'],\n    'subsector_id_38':['mean'],\n    'subsector_id_39':['mean'],\n    'subsector_id_40':['mean'],\n    'subsector_id_41':['mean'],\n    \n    'purchase_amount': ['sum', 'mean', 'max', 'min', 'std'],\n    'installments': ['sum', 'mean', 'max', 'min', 'std'],\n    'purchase_date': [np.ptp, 'min', 'max'],\n#     'month_lag': ['mean', 'max', 'min', 'std'],\n    'month_diff': ['mean', 'max', 'min', 'std'],\n        \n    }\n    \n    agg_transactions = data.groupby(['card_id']).agg(agg_func)\n    \n    agg_transactions.columns = ['_'.join(col).strip() for col in agg_transactions.columns.values]\n    agg_transactions.reset_index(inplace=True)\n    \n    temp_df = (data.groupby('card_id')\n          .size()\n          .reset_index(name='transactions_count'))\n    \n    agg_transactions = pd.merge(temp_df, agg_transactions, on='card_id', how='left')\n    \n    return agg_transactions\n\n\nagg_transactions = aggregate_transactions(transactions)\n# agg_transactions = reduce_mem_usage(agg_transactions)\n\nagg_transactions[:5]\n\ndel transactions\ngc.collect()\n# # scaling\n\nfrom sklearn.preprocessing import MinMaxScaler\nscaler = MinMaxScaler()\n\n\ncolumns_to_scale = [c for c in agg_transactions.columns if c not in ['card_id', 'Unnamed: 0']]\n\nagg_transactions[columns_to_scale ] = scaler.fit_transform(agg_transactions[columns_to_scale])\n\nprint(\"writing to csv.........\")\nagg_transactions.to_csv(r'/kaggle/working/agg_transactions.csv')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"agg_transactions.isna().sum()[:10]\nagg_transactions[:5]\nagg_transactions.to_csv(r'/kaggle/working/agg_transactions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# new=pd.DataFrame()\n# new.to_csv(r'/kaggle/working/transactions.csv')\nagg_transactions.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"raw","source":"def read_data(input_file):\n    df = pd.read_csv(input_file)\n    df['first_active_month'] = pd.to_datetime(df['first_active_month'])\n    df['elapsed_time'] = (datetime.date(2018, 2, 1) - df['first_active_month'].dt.date).dt.days\n    return df\n\n# train = read_data('../input/elo-dmbi-project/td005_1.csv')\ntrain = read_data('../input/elo-merchant-category-recommendation/train.csv')\nprint(\"training file read \")\n\ntrain = pd.merge(train, agg_transactions, on='card_id', how='left')\n# train.drop(columns=['Unnamed: 0'])\ntrain[['elapsed_time']] = scaler.fit_transform(train[['elapsed_time']])\n\n\n# indexNames = train[ abs(train['target']) > 10].index\n# train.drop(indexNames , inplace=True)\n\ntrain = pd.get_dummies(train, columns=['feature_1', 'feature_2'])\n\ntrain_split = np.split(train, [int(0.9*train.shape[0])] , axis=0)\ntrain_data = train_split[0]\ntest_data = train_split[1]\n\n\n\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_data(input_file):\n    df = pd.read_csv(input_file)\n    df['first_active_month'] = pd.to_datetime(df['first_active_month'])\n    df['elapsed_time'] = (datetime.date(2018, 2, 1) - df['first_active_month'].dt.date).dt.days\n    return df\n\n# train = read_data('../input/elo-dmbi-project/td005_1.csv')\ntrain = read_data('../input/elo-merchant-category-recommendation/train.csv')\nprint(\"training file read \")\n\ntrain = pd.merge(train, agg_transactions, on='card_id', how='left')\n# train.drop(columns=['Unnamed: 0'])\ntrain[['elapsed_time']] = scaler.fit_transform(train[['elapsed_time']])\n\n\nindexNames = train[ abs(train['target']) > 10].index\ntrain.drop(indexNames , inplace=True)\n\ntrain = pd.get_dummies(train, columns=['feature_1', 'feature_2'])\n\ntrain_split = np.split(train, [int(0.9*train.shape[0])] , axis=0)\ntrain_data = train_split[0]\ntest_data = train_split[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"#corelation matrix\n\n# f = plt.figure(figsize=(30, 30))\n# plt.matshow(train.corr(), fignum=f.number)\n# plt.xticks(range(train.shape[1]), train.columns, fontsize=14, rotation=45)\n# plt.yticks(range(train.shape[1]), train.columns, fontsize=14)\n# cb = plt.colorbar()\n# cb.ax.tick_params(labelsize=14)\n# plt.title('Correlation Matrix', fontsize=16);\n\n\n# print(train_data[\"target\"].corr(train_data[\"purchase_date_min\"]))\n\ncol_names = [c for c in train.columns if c not in ['card_id', 'first_active_month', 'Unnamed: 0','target']]\n\nfor colm in col_names:\n    print(colm ,\" :  \", train[\"target\"].corr(train[colm]) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"features_sorted = data=best_features.sort_values(by=\"importance\",\n                                           ascending=False)\nfeatures_sorted = features_sorted[\"feature\"].to_numpy()\n\nfor feat in features_sorted:\n    print(feat ,\" :  \", train_data[\"target\"].corr(train_data[feat]) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"param = {'num_leaves': 105,\n         'min_data_in_leaf': 66, \n         'objective':'regression',\n         'max_depth': 8,\n         'learning_rate': 0.005,\n         \"boosting\": \"gbdt\",\n         \"feature_fraction\": 0.733,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.9597 ,\n         \"bagging_seed\": 11,\n         \"metric\": 'rmse',\n         \"lambda_l1\": 2.251,\n         \"random_state\": 133,\n         \"verbosity\": -1}\n\n\n\n#features\nfeatures = [c for c in train.columns if c not in ['card_id', 'first_active_month', 'Unnamed: 0','target']]\n\ncategorical_feats = ['feature_1','feature_2']\n\ntrain_split = np.split(train, [int(0.7*train_data.shape[0])] , axis=0)\n\ntrain_data = train_split[0]\nvalidation_data = train_split[1]\n\n\nprint(train.shape[0])\nprint(train_data.shape[0])\nprint(validation_data.shape[0])\n\n\ntrn_data = lgb.Dataset(train_data[features],\n                           label=train_data[\"target\"],\n                           categorical_feature=categorical_feats\n                          )\n\nval_data = lgb.Dataset(validation_data[features],\n                           label=validation_data[\"target\"],\n                           categorical_feature=categorical_feats\n                          )\n\n\nnum_round = 10000\n    \nclf = lgb.train(param,\n                    trn_data,\n                    num_round,\n                    valid_sets = [trn_data, val_data],\n                    verbose_eval=100,\n                early_stopping_rounds = 100\n               )\n\n# print(features)\n\n####plot feature importance\n\nfeature_importance_df = pd.DataFrame()\n\nfold_importance_df = pd.DataFrame()\nfold_importance_df[\"feature\"] = features\nfold_importance_df[\"importance\"] = clf.feature_importance()\nfold_importance_df[\"fold\"] = 1\nfeature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n\n    \n\ncols = (feature_importance_df[[\"feature\", \"importance\"]]\n        .groupby(\"feature\")\n        .mean()\n        .sort_values(by=\"importance\", ascending=False)[:1000].index)\n\nbest_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)]\n\n\nplt.figure(figsize=(14,25))\nsns.barplot(x=\"importance\",\n            y=\"feature\",\n            data=best_features.sort_values(by=\"importance\",\n                                           ascending=False))\nplt.title('LightGBM Features')\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"param = {'num_leaves': 105,\n         'min_data_in_leaf': 66, \n         'objective':'regression',\n         'max_depth': 8,\n         'learning_rate': 0.005,\n         \"boosting\": \"rf\",\n         \"feature_fraction\": 0.733,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.9597 ,\n         \"bagging_seed\": 11,\n         \"metric\": 'rmse',\n         \"lambda_l1\": 2.251,\n         \"random_state\": 133,\n         \"verbosity\": -1}\n\n\n\nfrom sklearn.model_selection import KFold\nfolds = KFold(n_splits=5, shuffle=True, random_state=9)\n\noof = np.zeros(len(train_data))\npredictions = np.zeros(len(test_data))\n\nfeature_importance_df = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train_data.values, train_data[\"target\"].values)):\n    print(\"fold n°{}\".format(fold_))\n    trn_data = lgb.Dataset(train_data.iloc[trn_idx][features],\n                           label=train_data[\"target\"].iloc[trn_idx],\n                             )\n    val_data = lgb.Dataset(train_data.iloc[val_idx][features],\n                           label=train_data[\"target\"].iloc[val_idx],\n                            )\n\n    num_round = 10000\n    clf = lgb.train(param,\n                    trn_data,\n                    num_round,\n                    valid_sets = [trn_data, val_data],\n                    verbose_eval=100,\n                    early_stopping_rounds = 200)\n    \n    oof[val_idx] = clf.predict(train_data.iloc[val_idx][features], num_iteration=clf.best_iteration)\n    \n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"feature\"] = features\n    fold_importance_df[\"importance\"] = clf.feature_importance()\n    fold_importance_df[\"fold\"] = fold_ + 1\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    \n    predictions += clf.predict(test_data[features], num_iteration=clf.best_iteration) / folds.n_splits\n\nprint(\"CV score: {:<8.5f}\".format(mean_squared_error(oof, train_data[\"target\"])**0.5))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols = (feature_importance_df[[\"feature\", \"importance\"]]\n        .groupby(\"feature\")\n        .mean()\n        .sort_values(by=\"importance\", ascending=False)[:1000].index)\n\nbest_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)]\n\nplt.figure(figsize=(14,25))\nsns.barplot(x=\"importance\",\n            y=\"feature\",\n            data=best_features.sort_values(by=\"importance\",\n                                           ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom sklearn.metrics import mean_squared_error\nerrors_new = clf.predict(test_data[features]) - test_data[\"target\"] \n\nerrors_new = errors_new/ np.std(errors_new)\nplt.plot(errors_new ,'bo')\n\nprint('val RMSE:', mean_squared_error(test_data[\"target\"],  clf.predict(test_data[features]), squared=False) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(np.count_nonzero(abs(errors_new) > 2.5)/errors_new.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_train_data  = train_data.fillna(0.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_train_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# linear regressin - ordinary least squares \nfeatures = [c for c in train.columns if c not in ['card_id', 'first_active_month', 'Unnamed: 0','target']]\n\n# train_split = np.split(train, [int(0.8*train.shape[0])] , axis=0)\n\n# train_data = train_split[0]\n# validation_data = train_split[1]\n\nnew_train_data  = train_data.fillna(0.0)\nnew_val_data  = test_data.fillna(0.0)\n\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import mean_absolute_error\n\nmodel = LinearRegression()\nmodel.fit(new_train_data[features].to_numpy(), new_train_data[\"target\"].to_numpy())\n\n#predictions\npredictions = model.predict(new_train_data[features].to_numpy())\n# Calculate the absolute errors\nerrors = predictions - new_train_data[\"target\"].to_numpy()\n\nprint(\"Rsq\", model.score(new_train_data[features].to_numpy(), new_train_data[\"target\"].to_numpy()))\nprint(\"co-efficients\", model.coef_)\n\nprint('val RMSE:', mean_squared_error(new_train_data[\"target\"], predictions, squared=False) )\nprint('val mean absolute error:', mean_absolute_error(new_train_data[\"target\"], predictions) )\n# print(predictions)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from yellowbrick.regressor import cooks_distance\nfrom yellowbrick.regressor import ResidualsPlot\n\ncooks_distance(\n    new_train_data[features].to_numpy(), new_train_data[\"target\"].to_numpy(),\n    draw_threshold=True,\n    linefmt=\"C0-\", markerfmt=\",\"\n)\n\nvisualizer = ResidualsPlot(model)\n\nvisualizer.fit(new_train_data[features], new_train_data[\"target\"])  # Fit the training data to the visualizer\n# visualizer.score(X_test, y_test)  # Evaluate the model on the test data\nvisualizer.show()                 # Finalize and render the figure\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.plot(errors ,'.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import statsmodels.api as sm\n\n# mod_fit = sm.OLS(new_train_data[\"target\"],new_train_data[features]).fit()\nres = mod_fit.resid # residuals\nfig = sm.qqplot(res,line='45')\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import statsmodels.api as sm\n\n#Fit linear model to any dataset\nmodel_2 = sm.OLS(new_train_data[\"target\"],new_train_data[features])\nresults = model_2.fit()\n\n#create instance of influence\ninfluence = results.get_influence()\n\n#leverage (hat values)\nleverage = influence.hat_matrix_diag\n\n#Cook's D values (and p-values) as tuple of arrays\ncooks_d = influence.cooks_distance\n\n#standardized residuals\nstandardized_residuals = influence.resid_studentized_internal\n\nplt.plot(standardized_residuals ,'.')\nplt.plot(cooks_d[0] ,'.')\n\n# qq plot\nres = mod_fit.resid # residuals\nfig = sm.qqplot(res,line='45')\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(np.count_nonzero(abs(standardized_residuals) > 2.5)/standardized_residuals.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#linear regression - random forrests\nfeatures = [c for c in train.columns if c not in ['card_id', 'first_active_month', 'Unnamed: 0','target']]\n\ntrain_split = np.split(train, [int(0.8*train.shape[0])] , axis=0)\n\ntrain_data = train_split[0]\nvalidation_data = train_split[1]\n\nnew_train_data  = train_data.fillna(0.0)\nnew_val_data  = validation_data.fillna(0.0)\n\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import mean_absolute_error\n\n# Instantiate model with 1000 decision trees\nrf = RandomForestRegressor(n_estimators = 1000, random_state = 42)\n# Train the model on training data\n\nrf.fit(new_train_data[features].to_numpy(), new_train_data[\"target\"].to_numpy())\n\n# Use the forest's predict method on the test data\npredictions = rf.predict(new_val_data[features].to_numpy())\n# Calculate the absolute errors\nerrors = predictions - new_val_data[\"target\"].to_numpy()\n\n# Print out the RMSE\nprint('val RMSE:', mean_squared_error(new_val_data[\"target\"], predictions, squared=False) )\nprint('val mean absolute error:', mean_absolute_error(new_val_data[\"target\"], predictions, squared=False) )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"#cooks distance - linear regression\nfrom yellowbrick.regressor import cooks_distance\ncooks_distance(\n    new_train_data[features].to_numpy(), new_train_data[\"target\"].to_numpy(),\n    draw_threshold=True,\n    linefmt=\"C0-\", markerfmt=\",\"\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"#residual plots  - linear regression\nimport matplotlib.pyplot as plt\n\nplt.plot(errors ,'bo')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#XGBoost\n\nfrom xgboost import XGBRegressor\n\nmy_model = XGBRegressor(n_estimators=10000, learning_rate=0.005, num_parallel_tree=2,max_depth=10)\n\n# Add silent=True to avoid printing out updates with each cycle\nmy_model.fit(new_train_data[features].to_numpy(), new_train_data[\"target\"].to_numpy(), verbose=False, early_stopping_rounds=500,  eval_set=[(new_val_data[features].to_numpy(), new_val_data['target'].to_numpy())])\n\npredictions = my_model.predict(new_val_data[features].to_numpy())\n\nprint(\"RMSE : \" + str(mean_squared_error(predictions, new_val_data[\"target\"] )))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = my_model.predict(new_train_data[features].to_numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = pd.DataFrame(data=predictions,columns=['predictions'])\nresults[\"target\"] = new_val_data['target']\nresults","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## feature importance - comparison with other models - not working fully  \n\ndef plot_feature_importances(clf, X_train, y_train=None, \n                             top_n=20, figsize=(20,20), print_table=False, title=\"Feature Importances\"):\n     \n    __name__ = \"plot_feature_importances\"\n    \n\n    import matplotlib.pyplot as plt\n    \n    from xgboost.core     import XGBoostError\n    from lightgbm.sklearn import LightGBMError\n    \n    try: \n        if not hasattr(clf, 'feature_importances_'):\n            clf.fit(X_train.values, y_train.values.ravel())\n\n            if not hasattr(clf, 'feature_importances_'):\n                raise AttributeError(\"{} does not have feature_importances_ attribute\".\n                                    format(clf.__class__.__name__))\n                \n    except (XGBoostError, LightGBMError, ValueError):\n        clf.fit(X_train.values, y_train.values.ravel())\n            \n    feat_imp = pd.DataFrame({'importance':clf.feature_importances_})    \n    feat_imp['feature'] = X_train.columns\n    feat_imp.sort_values(by='importance', ascending=False, inplace=True)\n    feat_imp = feat_imp.iloc[:top_n]\n    \n    feat_imp.sort_values(by='importance', inplace=True)\n    feat_imp = feat_imp.set_index('feature', drop=True)\n    feat_imp.plot.barh(title=title, figsize=figsize)\n    plt.xlabel('Feature Importance Score')\n    plt.show()\n    \n    if print_table:\n        from IPython.display import display\n        print(\"Top {} features in descending order of importance\".format(top_n))\n        display(feat_imp.sort_values(by='importance', ascending=False))\n        \n    return feat_imp\n\nX_train = train_data[features].fillna(0)\ny_train = pd.DataFrame(train_data[\"target\"]).fillna(0)\n\nfrom xgboost              import XGBClassifier\nfrom sklearn.ensemble     import ExtraTreesClassifier\nfrom sklearn.tree         import ExtraTreeClassifier\nfrom sklearn.tree         import DecisionTreeClassifier\nfrom sklearn.ensemble     import GradientBoostingClassifier\nfrom sklearn.ensemble     import BaggingClassifier\nfrom sklearn.ensemble     import AdaBoostClassifier\nfrom sklearn.ensemble     import RandomForestClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom lightgbm             import LGBMClassifier\n\n\nclfs = [ RandomForestClassifier()]\n    \n#     LogisticRegression(),\n# #         LGBMClassifier(), \n#         BaggingClassifier(), GradientBoostingClassifier(), AdaBoostClassifier(),  RandomForestClassifier(), XGBClassifier(),]\n\nfor clf in clfs:\n    try:\n        _ = plot_feature_importances(clf, X_train, y_train, top_n=X_train.shape[1], title=clf.__class__.__name__)\n    except AttributeError as e:\n        print(e)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#check what is happening here\n\ndef aggregate_per_month(history):\n    grouped = history.groupby(['card_id', 'month_lag'])\n\n    agg_func = {\n            'purchase_amount': ['count', 'sum', 'mean', 'min', 'max', 'std'],\n            'installments': ['count', 'sum', 'mean', 'min', 'max', 'std'],\n            }\n\n    intermediate_group = grouped.agg(agg_func)\n    intermediate_group.columns = ['_'.join(col).strip() for col in intermediate_group.columns.values]\n    intermediate_group.reset_index(inplace=True)\n\n    final_group = intermediate_group.groupby('card_id').agg(['mean', 'std'])\n    final_group.columns = ['_'.join(col).strip() for col in final_group.columns.values]\n    final_group.reset_index(inplace=True)\n    \n    return final_group\n#___________________________________________________________\nfinal_group =  aggregate_per_month(transactions) \nfinal_group[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom datetime import datetime\nimport gc\nimport warnings\nfrom bayes_opt import BayesianOptimization\n\nimport numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nfrom sklearn.model_selection import KFold\nimport warnings\nimport time\nimport sys\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import mean_squared_error\nfrom datetime import datetime\nwarnings.simplefilter(action='ignore', category=FutureWarning)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def timer(start_time=None):\n    if not start_time:\n        start_time = datetime.now()\n        return start_time\n    elif start_time:\n        thour, temp_sec = divmod(\n            (datetime.now() - start_time).total_seconds(), 3600)\n        tmin, tsec = divmod(temp_sec, 60)\n        print('\\n Time taken: %i hours %i minutes and %s seconds.' % (thour, tmin, round(tsec, 2)))\n        \n\ndef reduce_mem_usage(df):\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/elo-dmbi-project/td005_1.csv\", index_col=0)\ntrain = reduce_mem_usage(train)\n\ntarget = train['target']\ndel train['target']\n\nfeatures = [c for c in train.columns if c not in ['card_id', 'first_active_month']]\n#features = [f for f in features if f not in unimportant_features]\ncategorical_feats = [c for c in features if 'feature_' in c]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def LGB_CV(\n          max_depth,\n          num_leaves,\n          min_data_in_leaf,\n          feature_fraction,\n          bagging_fraction,\n          lambda_l1\n         ):\n    \n    folds = KFold(n_splits=5, shuffle=True, random_state=15)\n    oof = np.zeros(train.shape[0])\n\n    for fold_, (trn_idx, val_idx) in enumerate(folds.split(train.values, target.values)):\n        print(\"fold n°{}\".format(fold_))\n        trn_data = lgb.Dataset(train.iloc[trn_idx][features],\n                               label=target.iloc[trn_idx],\n                               categorical_feature=categorical_feats)\n        val_data = lgb.Dataset(train.iloc[val_idx][features],\n                               label=target.iloc[val_idx],\n                               categorical_feature=categorical_feats)\n    \n        param = {\n            'num_leaves': int(num_leaves),\n            'min_data_in_leaf': int(min_data_in_leaf), \n            'objective':'regression',\n            'max_depth': int(max_depth),\n            'learning_rate': 0.01,\n            \"boosting\": \"gbdt\",\n            \"feature_fraction\": feature_fraction,\n            \"bagging_freq\": 1,\n            \"bagging_fraction\": bagging_fraction ,\n            \"bagging_seed\": 11,\n            \"metric\": 'rmse',\n            \"lambda_l1\": lambda_l1,\n            \"verbosity\": -1\n        }\n    \n        clf = lgb.train(param,\n                        trn_data,\n                        10000,\n                        valid_sets = [trn_data, val_data],\n                        verbose_eval=500,\n                        early_stopping_rounds = 200)\n        \n        oof[val_idx] = clf.predict(train.iloc[val_idx][features],\n                                   num_iteration=clf.best_iteration)\n        \n        del clf, trn_idx, val_idx\n        gc.collect()\n        \n    return -mean_squared_error(oof, target)**0.5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LGB_BO = BayesianOptimization(LGB_CV, {\n    'max_depth': (4, 10),\n    'num_leaves': (5, 130),\n    'min_data_in_leaf': (10, 150),\n    'feature_fraction': (0.7, 1.0),\n    'bagging_fraction': (0.7, 1.0),\n    'lambda_l1': (0, 6)\n    })","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('-'*126)\n\nstart_time = timer(None)\nwith warnings.catch_warnings():\n    warnings.filterwarnings('ignore')\n    LGB_BO.maximize(init_points=2, n_iter=20, acq='ei', xi=0.0)\ntimer(start_time)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"|   iter    |  target   | baggin... | featur... | lambda_l1 | max_depth | min_da... | num_le... |\n22       | -4.349    |  0.9597   |  0.7335   |  2.251    |  7.133    |  66.01    |  105.2 ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}