{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"# Light GBM baseline\n\nHeavily taken from:  https://www.kaggle.com/sz8416/lb-1-4439-gacr-prediction-eda-lgb-baseline and from Olivier \n\n### Results and notes\n- Best LB score: 1.4378 using visitStartTime and local hour of day, highly tuned hyper parameters. CV score 1.6415 session and 1.6126 user level\n\n- Current last model: LB = 1.4404, CV = 1.6167, 1.5919, and visit start time + local hour of day + many categorical interaction features\n- Top 10 categorical interactions, removed visitStartTime, added year, day of week, time since last session:\n        - Session level CV score:  1.6242508497927641\n        - User level CV score:  1.6031847639160595\n"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport time\nimport warnings\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nplt.style.use('ggplot')\n\n#Sklearn\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold\n\nwarnings.filterwarnings('ignore')\n\n#Light GBM\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36be15bf4163a0fb640f3a944b71b783335251ba"},"cell_type":"code","source":"os.listdir('../input/kernel-for-file-processing-2')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c148d09cbdceb72be34745e3c30fad0b3df04bf9"},"cell_type":"code","source":"# Extract target values and Ids\ncat_cols = ['channelGrouping',\n            'device.browser',\n       'device.deviceCategory', 'device.isMobile', 'device.operatingSystem',\n       'geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country',\n       'geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region',\n       'geoNetwork.subContinent','trafficSource.adContent',\n       #'trafficSource.adwordsClickInfo.adNetworkType',\n       'trafficSource.adwordsClickInfo.gclId',\n       #'trafficSource.adwordsClickInfo.isVideoAd',\n       #'trafficSource.adwordsClickInfo.page',\n       #'trafficSource.adwordsClickInfo.slot', #Drop as only 3 values and always poor\n       'trafficSource.campaign',\n       'trafficSource.isTrueDirect', 'trafficSource.keyword',\n       'trafficSource.medium', 'trafficSource.referralPath',\n       'trafficSource.source',\n        #Interaction columns\n        'geoNetwork.city+geoNetwork.networkDomain',\n        'device.operatingSystem+geoNetwork.networkDomain',\n        'device.operatingSystem+geoNetwork.city', \n        'channelGrouping+geoNetwork.networkDomain',\n        'geoNetwork.city+trafficSource.source',\n        'geoNetwork.networkDomain+trafficSource.source',\n        'geoNetwork.networkDomain+trafficSource.referralPath',\n        'geoNetwork.networkDomain+trafficSource.medium',\n        'geoNetwork.city+trafficSource.medium',\n        'geoNetwork.city+geoNetwork.country',\n        #Time columns (categorical)\n        # '_dayofweek', '_year', '_local_hourofday' #These are time related but actually integer  categories\n           ]\n\nto_drop = ['trafficSource.adwordsClickInfo.adNetworkType','trafficSource.adwordsClickInfo.isVideoAd',\n          'trafficSource.adwordsClickInfo.page','trafficSource.adwordsClickInfo.slot']\n\nnum_cols = ['visitNumber', 'totals.bounces', 'totals.hits',\n            'totals.newVisits', 'totals.pageviews', \n            '_prev_totals.bounces_1', '_prev_totals.bounces_2',\n           '_next_totals.bounces_1', '_next_totals.bounces_2',\n           '_prev_totals.hits_1', '_prev_totals.hits_2', '_next_totals.hits_1',\n           '_next_totals.hits_2', '_prev_totals.pageviews_1',\n           '_prev_totals.pageviews_2', '_next_totals.pageviews_1',\n           '_next_totals.pageviews_2', '_prev__local_hourofday_1',\n           '_prev__local_hourofday_2', '_next__local_hourofday_1',\n           '_next__local_hourofday_2', '_difference_first_last',\n           '_time_since_first_visit', 'visitNumber_12H', 'visitNumber_7D',\n           'visitNumber_30D' ]\n\n\nvisitStartTime = ['visitStartTime']\n\ntime_shift_cols = ['_time_since_last_visit', '_time_since_last_visit_2',\n       '_time_to_next_visit', '_time_to_next_visit_2']\n        #[ '_time_since_first_visit' ] # '_difference_first_last'] #These are timedelta format which require additional steps\n\ntime_cols = [ '_local_hourofday' , '_dayofweek', '_year']\n\n\nID_cols = ['date', 'fullVisitorId', 'sessionId', 'visitId']\n\ntarget_col = ['totals.transactionRevenue']\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1df87f4b40a4204e3a57c585548e042097bf73e7"},"cell_type":"code","source":"%%time\n#del train_df\n#del test_df\n\ntrain_df = pd.read_pickle('../input/kernel-for-file-processing-2/train_flat_FE_CAT_LE.pkl')\ntest_df = pd.read_pickle('../input/kernel-for-file-processing-2/test_flat_FE_CAT_LE.pkl')\n\n#train_df['_time_since_last_visit'] = pd.to_numeric(train_df['_time_since_last_visit'])\n#test_df['_time_since_last_visit'] = pd.to_numeric(test_df['_time_since_last_visit'])\n\ntrain_df.drop(to_drop, axis = 1, inplace = True)\ntest_df.drop(to_drop, axis = 1, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb52a499beb09ebf96562562938a51904443df98"},"cell_type":"code","source":"#Time features\ntrain_df['_dayofweek'] = train_df['visitStartTime'].dt.dayofweek\ntrain_df['_year'] = train_df['visitStartTime'].dt.year\n\ntest_df['_dayofweek'] = test_df['visitStartTime'].dt.dayofweek\ntest_df['_year'] = test_df['visitStartTime'].dt.year","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"217762bb0198f7e2b8e94c52e695275a13a20c41"},"cell_type":"code","source":"train_df.groupby('fullVisitorId').count().info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0251a53c7ee42e4f25ea35530b903410b4c2822f"},"cell_type":"code","source":"%%time\n#Numeric as float\nfor n in [num_cols + time_cols]:\n    train_df[n] = train_df[n].fillna(0).astype('int')\n    test_df[n] = test_df[n].fillna(0).astype('int')\n\n#train totals.transactionRevenue\ntrain_df['totals.transactionRevenue'] = train_df['totals.transactionRevenue'].fillna(0).astype('float')\n\n#visitStartTime\nfor v in time_shift_cols:\n    train_df[v] = pd.to_numeric(train_df[v]) / 1e9 # in seconds\n    test_df[v] = pd.to_numeric(test_df[v]) / 1e9\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de8a71b7646479655df746e963ce47bb6b74f996"},"cell_type":"code","source":"#Index\ntrain_idx = train_df['fullVisitorId']\ntest_idx = test_df['fullVisitorId']\n\n#Targets\ntrain_target = np.log1p(train_df.groupby(\"fullVisitorId\")[\"totals.transactionRevenue\"].sum())\ntrain_y = np.log1p(train_df[\"totals.transactionRevenue\"])\n\n#Datasets\ntrain_X = train_df[cat_cols + num_cols + time_cols + time_shift_cols]\ntest_X = test_df[cat_cols + num_cols + time_cols + time_shift_cols]\n\n\nprint(train_X.shape)\nprint(test_X.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"682799439737ad71547b9a7fc21595cbdb4dc4de"},"cell_type":"code","source":"train_X.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd7e23e5fc034417c1230c0c124e96abbd7aecc1"},"cell_type":"code","source":"predictions_train = pd.DataFrame(data = {'fullVisitorId':train_df['fullVisitorId'], \n                                         'sessionId':train_df['sessionId'], \n                                         'visitId':train_df['visitId'],\n                                         'index':train_df.index,\n                                         'totals.transactionRevenue':np.log1p(train_df['totals.transactionRevenue']),\n                                         'predictedRevenue':np.nan})\n\npredictions_test = pd.DataFrame(data = {'fullVisitorId':test_df['fullVisitorId'], \n                                         'sessionId':test_df['sessionId'], \n                                         'visitId':test_df['visitId'],\n                                         'index':test_df.index, \n                                         'predictedRevenue':np.nan})\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd92bc4e54660d821b6f5d3ad0b8ebce2cb13c0b"},"cell_type":"code","source":"from sklearn.model_selection import GroupKFold\n\ndef get_folds(df=None, n_splits=5):\n    \"\"\"Returns dataframe indices corresponding to Visitors Group KFold\"\"\"\n    # Get sorted unique visitors\n    unique_vis = np.array(sorted(df['fullVisitorId'].unique()))\n\n    # Get folds\n    folds = GroupKFold(n_splits=n_splits)\n    fold_ids = []\n    ids = np.arange(df.shape[0])\n    for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):\n        fold_ids.append(\n            [\n                ids[df['fullVisitorId'].isin(unique_vis[trn_vis])],\n                ids[df['fullVisitorId'].isin(unique_vis[val_vis])]\n            ]\n        )\n\n    return fold_ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"624080b702473533334f1411fae0ad83d649a20f"},"cell_type":"code","source":"from lightgbm import LGBMRegressor\n\n#Initialize LGBM\ngbm = LGBMRegressor(objective = 'regression', \n                     boosting_type = 'gbdt', \n                     metric = 'rmse',\n                     n_estimators = 10000, #10000\n                     num_leaves = 30,\n                     learning_rate = 0.01, #0.01\n                     bagging_fraction = 0.9,#0.8\n                     feature_fraction = 0.3,#.3\n                     #bagging_seed = 42,\n                     #max_depth = 12, #-1 \n                     #categorical_feature = [train_df[cat_cols].columns.get_loc(c) for c in cat_cols ] ,\n                     #cat_smooth = 20\n                    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca5bbe1f19c78b4135f66f1874114c32389228ce"},"cell_type":"code","source":"#predictions_train.info()\npredictions_train.groupby('fullVisitorId').count().info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99b6b3c4865de8b41cde4bbcaa98cced42f7aebc","_kg_hide-output":true},"cell_type":"code","source":"%%time\n#Initilization\nall_K_fold_results = []\n#kf = KFold(n_splits=5, shuffle = True)\nfolds = get_folds(df=train_df, n_splits=5)\nk = 0\n\n\nfor fold_, (d, v) in enumerate(folds):\n    dev_index, val_index  = train_X.index[d], train_X.index[v]\n    X_dev, X_val = train_X.loc[dev_index], train_X.loc[val_index]\n    y_dev, y_val = train_y[dev_index], train_y[val_index]\n    \n    #Fit the model\n    model = gbm.fit(X_dev,y_dev, eval_set=[(X_val, y_val)],verbose = 100, \n                    eval_metric = 'rmse', early_stopping_rounds = 100) #100\n    \n    #Predict out of fold \n    predictions_train.loc[val_index, 'predictedRevenue'] = gbm.predict(X_val, num_iteration= model.best_iteration_).copy()\n    predictions_train[predictions_train['predictedRevenue'] < 0]['predictedRevenue'] = 0\n    print(predictions_train.groupby('fullVisitorId').count().info())\n    \n    #Predict on train using all train for each fold\n    sub_prediction_train = pd.Series(data = gbm.predict(train_X, num_iteration= model.best_iteration_)).copy()\n    sub_prediction_train[sub_prediction_train<0] = 0\n    predictions_train['Predictions_{}'.format(k)] = sub_prediction_train.values.copy()\n    print(predictions_train.groupby('fullVisitorId').count().info())\n    \n    #Predict on test set based on current fold model. Average results\n    sub_prediction = pd.Series(data = gbm.predict(test_X, num_iteration= model.best_iteration_))\n    sub_prediction[sub_prediction<0] = 0\n    predictions_test['Predictions_{}'.format(k)] = sub_prediction.copy()\n    k += 1 #increase by 1\n    \n    #Save current fold values\n    fold_results = {'best_iteration_' : model.best_iteration_, \n                   'best_score_' : model.best_score_['valid_0']['rmse'], \n                   'evals_result_': model.evals_result_['valid_0']['rmse'],\n                   'feature_importances_' : model.feature_importances_}\n\n    all_K_fold_results.append(fold_results.copy())\n    \n\n#Save results\nresults = pd.DataFrame(all_K_fold_results)\npredictions_test['average_predictions'] = predictions_test.iloc[:,-5:].mean(axis = 1).copy()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3a53cb583612d114b39e540fc56935fa80bcac3"},"cell_type":"code","source":"def RMSE_log_sum(pred, df):\n    #set negative values to zero\n    pred[pred < 0] = 0\n    \n    #Build new dataframe\n    pred_df = pd.DataFrame(data = {'fullVisitorId': df['fullVisitorId'].values, \n                                       'transactionRevenue': df['totals.transactionRevenue'].values,\n                                      'predictedRevenue':np.expm1(pred) })\n    #Compute sum\n    pred_df = pred_df.groupby('fullVisitorId').sum().reset_index()\n\n    mse_log_sum = mean_squared_error( np.log1p(pred_df['transactionRevenue'].values), \n                             np.log1p(pred_df['predictedRevenue'].values)  )\n\n    #print('log (sum + 1): ',np.sqrt(mse_log_sum))\n    return np.sqrt(mse_log_sum)\n\n\ndef save_submission(pred_test, test_df, file_name):\n    #Zero negative predictions\n    pred_test[pred_test < 0] = 0\n    \n    #Create temporary dataframe\n    sub_df = pd.DataFrame(data = {'fullVisitorId':test_df['fullVisitorId'], \n                             'predictedRevenue':np.expm1(pred_test)})\n    sub_df = sub_df.groupby('fullVisitorId').sum().reset_index()\n    sub_df.columns = ['fullVisitorId', 'predictedLogRevenue']\n    sub_df['predictedLogRevenue'] = np.log1p(sub_df['predictedLogRevenue'])\n    sub_df.to_csv(file_name, index = False)\n\n    \ndef visualize_results(results):\n#Utility function to plot fold loss and best model feature importance\n    plt.figure(figsize=(16, 12))\n\n    #----------------------------------------\n    # Plot validation loss\n    plt.subplot(2,2,1)\n\n    for K in range(results.shape[0]):\n        plt.plot(np.arange(len(results.evals_result_[K])), results.evals_result_[K], label = 'fold {}'.format(K))\n\n    plt.xlabel('Boosting iterations')\n    plt.ylabel('RMSE')\n    plt.title('Validation loss vs boosting iterations')\n    plt.legend()\n\n    #----------------------------------------\n    # Plot box plot of RMSE\n    plt.subplot(2, 2, 2)    \n    scores = results.best_score_\n    plt.boxplot(scores)\n    rmse_mean = np.mean(scores)\n    rmse_std = np.std(scores)\n    plt.title('RMSE Mean:{:.3f} Std: {:.4f}'.format(rmse_mean,rmse_std ))\n    \n    #----------------------------------------\n    # Plot feature importance\n    #feature_importance = results.sort_values('best_score_').feature_importances_[0]\n    df_feature_importance = pd.DataFrame.from_records(results.feature_importances_)\n    feature_importance = df_feature_importance.mean()\n    std_feature_importance = df_feature_importance.std()\n    \n    # make importances relative to max importance\n    #feature_importance = 100.0 * (mean_feature_importance / mean_feature_importance.sum())\n    sorted_idx = np.argsort(feature_importance)\n    pos = np.arange(sorted_idx.shape[0]) + .5\n    plt.subplot(2, 1, 2)\n    plt.bar(pos, feature_importance[sorted_idx], align='center', yerr = std_feature_importance)\n    xlabels = [ test_X.columns.values[i] for i in sorted_idx]\n    plt.xticks(pos, xlabels, rotation = 90)\n    plt.xlabel('Feature')\n    plt.ylabel('Avg Importance score')\n    plt.title('Mean Feature Importance over K folds') \n    \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"25dd9b6ff2410a4871060febfd5cc8162e88c258"},"cell_type":"markdown","source":"### Previous iteration\nSession level CV score:  1.6112586662249373\nUser level CV score:  1.5919866631295672"},{"metadata":{"trusted":true,"_uuid":"4e0a0a727f93a0a4066ba4abea0e679ca396be2e"},"cell_type":"code","source":"print('Session level CV score: ', np.mean(results.best_score_))\n#print('Session level CV score (all data):', np.sqrt(mean_squared_error(predictions_train['totals.transactionRevenue'], predictions_train['predictedRevenue'])))\nprint('User level CV score: ', RMSE_log_sum(predictions_train['predictedRevenue'],train_df))\nprint(predictions_train.groupby('fullVisitorId').count().info())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71043451b5aeb4cd2b97cc9bb21b17889899a4af"},"cell_type":"code","source":"visualize_results(results)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f5f6109b9e773ba4ca78f9aae4c4111302abbbb"},"cell_type":"code","source":"train_df['visitStartTime'] = pd.to_datetime(train_df['visitStartTime'])\n\nerror_df = pd.DataFrame(data = {'visitStartTime':train_df['visitStartTime'],'fullVisitorId':train_df['sessionId'], \n                                'True_log_revenue' : np.log1p(train_df['totals.transactionRevenue']), \n                                'Predicted_log_revenue':predictions_train['predictedRevenue']  })\n\nerror_df['Difference'] = error_df['True_log_revenue'] - error_df['Predicted_log_revenue']\nerror_df['True_is_non_zero'] = error_df['True_log_revenue'] > 0\n#temp_df.columns = ['fullVisitorId', 'predictedLogRevenue']\n#sub_df['predictedLogRevenue'] = np.log1p(sub_df['predictedLogRevenue'])\n#sub_df.to_csv(file_name, index = False)\nerror_df.head(5).sort_values('True_log_revenue')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f1bbf04ac798d53a163ae778df36002e2f02361"},"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize = (20,7))\n\n\nsns.distplot(error_df[error_df['True_is_non_zero'] == False]['True_log_revenue'], ax = ax1, label = 'true')\nsns.distplot(error_df[error_df['True_is_non_zero'] == False ]['Predicted_log_revenue'], ax = ax1, label = 'pred')\nax1.legend()\nax1.set_ylim(0,.1)\nax1.set_xlabel('Log revenue (session)')\nax1.set_title('Distribution of log revenues for sessions with zero true revenue ')\n\nsns.distplot(error_df[error_df['True_is_non_zero'] == True]['True_log_revenue'], ax = ax2, label = 'true')\nsns.distplot(error_df[error_df['True_is_non_zero'] == True ]['Predicted_log_revenue'], ax = ax2, label = 'pred')\nax2.legend()\nax2.set_ylim(0,.5)\nax2.set_xlabel('Log revenue (session)')\nax2.set_title('Distribution of log revenues for sessions with non zero true revenue ')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d47365c65551cf87e0848b22e93058e95efa01c"},"cell_type":"code","source":"sorted_non_zero = error_df[error_df['True_is_non_zero'] == True].sort_values('visitStartTime')\n\n\nplt.figure(figsize = (20,15))\nplt.subplot(2,2,1)\nplt.plot(sorted_non_zero.visitStartTime, sorted_non_zero.True_log_revenue , label = 'True')\nplt.plot(sorted_non_zero.visitStartTime, sorted_non_zero.Predicted_log_revenue , alpha = .5, label = 'Pred')\nplt.title('Log revenue over time (non zero true sessions only)')\nplt.legend()\nplt.xlabel('Time: sessions')\n\nplt.subplot(2,2,2)\ndaily_error_non_zero_df = sorted_non_zero.set_index('visitStartTime', drop = True).resample('D').mean()\nplt.plot(daily_error_non_zero_df.index, daily_error_non_zero_df.True_log_revenue , label = 'True')\nplt.plot(daily_error_non_zero_df.index, daily_error_non_zero_df.Predicted_log_revenue , label = 'Pred')\nplt.title('Daily average log revenue (non zero true sessions only)')\n\nplt.subplot(2,2,3)\nweekly_error_df = error_df.set_index('visitStartTime', drop = True).resample('W').mean()\nplt.plot(weekly_error_df.index, weekly_error_df.True_log_revenue , label = 'True')\nplt.plot(weekly_error_df.index, weekly_error_df.Predicted_log_revenue , label = 'Pred')\nplt.title('Weekly average log revenue (all session)')\n\n\nplt.subplot(2,2,4)\ndaily_error_df = error_df.set_index('visitStartTime', drop = True).resample('D').mean()\nplt.plot(daily_error_df.index, daily_error_df.True_log_revenue , label = 'True')\nplt.plot(daily_error_df.index, daily_error_df.Predicted_log_revenue , label = 'Pred')\nplt.title('Daily average log revenue (all session)')\n\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a6e665d2f3b0bc25d77e9457fb4c485be1104a9"},"cell_type":"code","source":"sorted_non_zero = error_df[error_df['True_is_non_zero'] == True].sort_values('visitStartTime')\nsorted_zero = error_df[error_df['True_is_non_zero'] == False].sort_values('visitStartTime')\n\n\nplt.figure(figsize = (20,5))\nplt.subplot(1,3,1)\nts_error_df = error_df.set_index('visitStartTime', drop = True)\ndifference_rev_df = error_df.sort_values('visitStartTime')\nplt.plot(error_df.visitStartTime, error_df.Difference , label = 'True - predicted', color = 'grey')\nplt.title('Train - Pred (log rev) for all sessions')\n\nplt.subplot(1,3,2)\nplt.plot(sorted_non_zero.visitStartTime, sorted_non_zero.Difference , label = 'True - predicted',\n         color = 'grey')\nplt.title('Train - Pred for non zero sessions only')\n\nplt.subplot(1,3,3)\nplt.plot(sorted_zero.visitStartTime, sorted_zero.Difference,\n         color = 'grey')\nplt.title('Train - Pred for zero sessions only')\n\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8220eae53b995a64062270f63512e284a6c69350"},"cell_type":"code","source":"sns.jointplot(x=\"True_log_revenue\", y=\"Predicted_log_revenue\", data=sorted_non_zero)\ndisplay('Joint distribution of log rev for non zero sessions only')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e48ae93631e92186cbb44b637f6b0386b4275ca1"},"cell_type":"code","source":"save_submission(predictions_test['average_predictions'], test_df, 'submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e7f056d64108c1fd8c048ff280f7ade40cfb06c"},"cell_type":"code","source":"predictions_train.to_csv('level_1_train_output.csv')\npredictions_test.to_csv('level_1_test_output.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e49790297f26fecdcfbca84ef11eeb90a69b1363"},"cell_type":"markdown","source":"## Best features "},{"metadata":{"trusted":true,"_uuid":"f5b4237b3a16e98db83ab4d91b0b3dd3d8934217"},"cell_type":"code","source":"feature_importance = results.feature_importances_.mean()\nsorted_idx = np.argsort(feature_importance)\nfeature_importance[sorted_idx]\nfeature_names = [ test_X.columns.values[i] for i in sorted_idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b70d6dd50c41d067a58445565c2ffd994be9003"},"cell_type":"code","source":"importance_df = pd.DataFrame({'features_names': feature_names, 'importance':feature_importance[sorted_idx] })\nimportance_df.sort_values('importance', ascending = False)['features_names'][0:20].values","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}