'']\n print(\"\\n mae_cv score: {:.4f} ({:.4f})\\n\".format( (-scores['test_mae']).mean(), scores['test_mae'].std() ))\n print(\"\\n rmsle_cv score: {:.4f} ({:.4f})\\n\".format( (np.sqrt(-scores['test_nmse'])).mean(), scores['test_nmse'].std() ))\n print(\"\\n r2_cv score: {:.4f} ({:.4f})\\n\".format( scores['test_r2'].mean(), scores['test_r2'].std() ))\n return (scores)\n\n# useful when you can't decide on parameter setting from best_params_\n# result_details(grid_result,'mean_test_nmse',100)\ndef result_details(grid_result,sorting='mean_test_nmse',cols=100):\n param_df = pd.DataFrame.from_records(grid_result.cv_results_['params'])\n param_df['mean_test_nmse'] = np.sqrt(-grid_result.cv_results_['mean_test_nmse'])\n param_df['std_test_nmse'] = np.sqrt(grid_result.cv_results_['std_test_nmse'])\n param_df['mean_test_mae'] = -grid_result.cv_results_['mean_test_mae']\n param_df['std_test_mae'] = -grid_result.cv_results_['std_test_mae']\n param_df['mean_test_r2'] = -grid_result.cv_results_['mean_test_r2']\n param_df['std_test_r2'] = -grid_result.cv_results_['std_test_r2']\n return param_df.sort_values(by=[sorting]).tail(cols)\n\ndef rmsle(y, y_pred):\n return np.sqrt(mean_squared_error(y, y_pred))\n\ndef mae(y, y_pred):\n return mean_absolute_error(y,y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:08:54.728789Z","iopub.execute_input":"2022-09-19T01:08:54.729249Z","iopub.status.idle":"2022-09-19T01:08:54.741878Z","shell.execute_reply.started":"2022-09-19T01:08:54.729214Z","shell.execute_reply":"2022-09-19T01:08:54.740919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lasso Regression Model
","metadata":{}},{"cell_type":"code","source":"# initialize the algorithm for the GridSearchCV function\nlasso = Lasso()\ntuningLasso = 1 # takes 2 minutes to complete\n\nif (tuningLasso == 1):\n # use this when tuning\n param_grid={\n 'alpha':[0.01,], # done, lower keeps getting better, but don't want to go too low and begin overfitting (alpha is related to L1 reg)\n 'fit_intercept':[True], # done, big difference\n 'normalize':[False], # done, big difference\n 'precompute':[False], # done, no difference\n 'copy_X':[True], # done, no difference\n 'max_iter':[200], # done\n 'tol':[0.05], # done, not much difference # was 0.005 but that would cause error: ConvergenceWarning: Objective did not converge\n 'warm_start':[False], # done, no difference\n 'positive':[False], # done, big difference\n 'random_state':[1],\n 'selection':['cyclic'] # done both are same, cyclic is default\n }\n\nelse:\n # use this when not tuning\n param_grid={\n 'alpha':[0.2], \n 'fit_intercept':[True],\n 'normalize':[False],\n 'precompute':[False],\n 'copy_X':[True],\n 'max_iter':[200],\n 'tol':[0.0001],\n 'warm_start':[False],\n 'positive':[False],\n 'random_state':[None],\n 'selection':['cyclic']\n }\n\nscorers = {\n 'r2': 'r2',\n 'nmsle': 'neg_mean_squared_log_error', \n 'nmse': 'neg_mean_squared_error',\n 'mae': 'neg_mean_absolute_error'\n}\n# To be used within GridSearch \ninner_cv = KFold(n_splits=4, shuffle=True, random_state=None)\n\n# To be used in outer CV \nouter_cv = KFold(n_splits=4, shuffle=True, random_state=None) \n\n#inner loop KFold example:\ngsc = GridSearchCV(\n estimator=lasso,\n param_grid=param_grid,\n #scoring='neg_mean_squared_error', # 'roc_auc', # or 'r2', etc\n scoring=scorers, # 'roc_auc', # or 'r2', etc -> can output several scores, but only refit to one. Others are just calculated\n #scoring='neg_mean_squared_error', # or look here for other choices \n # https://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter\n #cv=5,\n cv=inner_cv, # this will use KFold splitting to change train/test/validation datasets randomly\n verbose=0,\n return_train_score=True, # keep the other scores\n refit='nmse' # use this one for optimizing\n)\n\ngrid_result = runGSCV(2, train, y_train)\n\nrd = result_details(grid_result,'random_state',100)\nrd[['random_state','alpha','mean_test_nmse','std_test_nmse','mean_test_mae','std_test_mae','mean_test_r2','std_test_r2']].sort_values(by=['random_state','alpha'])\n","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:08:54.743396Z","iopub.execute_input":"2022-09-19T01:08:54.743870Z","iopub.status.idle":"2022-09-19T01:08:57.252456Z","shell.execute_reply.started":"2022-09-19T01:08:54.743837Z","shell.execute_reply":"2022-09-19T01:08:57.251017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tuning_lasso = 1\nlasso = make_pipeline(RobustScaler(), Lasso(alpha =0.0005, tol=0.05, random_state=1)) # was 1 \nlasso_new = make_pipeline(RobustScaler(), Lasso(**grid_result.best_params_))","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:08:57.254389Z","iopub.execute_input":"2022-09-19T01:08:57.254964Z","iopub.status.idle":"2022-09-19T01:08:57.268228Z","shell.execute_reply.started":"2022-09-19T01:08:57.254926Z","shell.execute_reply":"2022-09-19T01:08:57.266600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (tuning_lasso == 1):\n #TEMP\n model_results = [] # model flow, mae, rmsle\n models = [lasso, lasso_new]\n\n for model in models:\n #print(model)\n with MyTimer(): \n scores = calc_all_scores(model,5,5)\n #print(\"------------------------------------------\")\n model_results.append([model, (-scores['test_mae']).mean(), (np.sqrt(-scores['test_nmse'])).mean(), scores['test_r2'].mean()])\n\n df_mr = pd.DataFrame(model_results, columns = ['model','mae','rmsle','r2'])\n df_mr.sort_values(by=['rmsle'])","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:08:57.270137Z","iopub.execute_input":"2022-09-19T01:08:57.270661Z","iopub.status.idle":"2022-09-19T01:08:58.850840Z","shell.execute_reply.started":"2022-09-19T01:08:57.270623Z","shell.execute_reply":"2022-09-19T01:08:58.849574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lasso = make_pipeline(RobustScaler(), Lasso(alpha =0.0005, tol=0.05, random_state=1)) # was 1 \n\nif (tuning_lasso == 1):\n for i in [2,5,20,42,99]:\n from sklearn.linear_model import Lasso\n print('random_state =',i)\n\n l = {'alpha': 0.01, 'copy_X': True, 'fit_intercept': True, 'max_iter': 200, 'normalize': False, 'positive': False, 'precompute': False, 'selection': 'cyclic', 'tol': 0.05, 'warm_start': False}\n lasso_new = make_pipeline(RobustScaler(), Lasso(**l, random_state=i))\n #lasso_new = Lasso(**l, random_state=i)\n\n model_results = [] # model flow, mae, rmsle\n models = [lasso, lasso_new]\n\n for model in models:\n #print(model)\n with MyTimer(): \n scores = calc_all_scores(model,5,5)\n #print(\"------------------------------------------\")\n model_results.append([model, (-scores['test_mae']).mean(), (np.sqrt(-scores['test_nmse'])).mean(), scores['test_r2'].mean()])\n\n df_mr = pd.DataFrame(model_results, columns = ['model','mae','rmsle','r2'])\n print(df_mr.sort_values(by=['rmsle']))\nelse:\n lasso_new = make_pipeline(RobustScaler(), Lasso(**grid_result.best_params_, random_state=i))","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:08:58.857141Z","iopub.execute_input":"2022-09-19T01:08:58.860784Z","iopub.status.idle":"2022-09-19T01:09:07.091189Z","shell.execute_reply.started":"2022-09-19T01:08:58.860725Z","shell.execute_reply":"2022-09-19T01:09:07.087390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Elastic Net Regression Model
","metadata":{}},{"cell_type":"code","source":"# initialize the algorithm for the GridSearchCV function\nif (method == \"ensemble\"):\n enet_tol = 0.01 # or try 0.01 - default is 0.0001\n ENet = ElasticNet(tol=enet_tol) # added tol=0.05 to avoid errors\n tuningENet = 0 # takes 2 minutes to complete\n\n if (tuningENet == 1):\n # use this when tuning\n param_grid={\n 'alpha':[0.01],\n 'l1_ratio':[0.75,0.8,0.85,0.9],\n 'fit_intercept':[True], # ,False\n 'normalize':[False], # True,\n 'max_iter':range(350,450,50),\n 'selection':['random'], # 'cyclic',\n 'random_state':[3],\n 'tol':[enet_tol]\n }\n\n else:\n # use this when not tuning\n param_grid={\n 'alpha':[0.01],\n 'l1_ratio':[.9],\n 'fit_intercept':[True],\n 'normalize':[False],\n 'max_iter':[350], # default 1000\n 'selection':['random'],\n 'random_state':[3],\n 'tol':[enet_tol]\n }\n\nelse: # (method == \"stacked\")\n enet_tol = 0.0001 # or try 0.01 \n ENet = ElasticNet() # added tol=0.05 to avoid errors\n tuningENet = 0 # takes 2 minutes to complete\n\n if (tuningENet == 1):\n # use this when tuning\n param_grid={\n 'alpha':[0.01,0.05],\n 'l1_ratio':[0.8,0.85,0.9],\n 'fit_intercept':[True], # ,False\n 'normalize':[False], # True,\n 'max_iter':range(350,450,50),\n 'selection':['random'], # 'cyclic',\n 'random_state':[3],\n 'tol':[enet_tol]\n }\n\n else:\n # use this when not tuning\n param_grid={\n 'alpha':[0.05],\n 'l1_ratio':[.85],\n 'fit_intercept':[True],\n 'normalize':[False],\n 'max_iter':[500], # default 1000\n 'selection':['random'],\n 'random_state':[3],\n 'tol':[enet_tol]\n }\n\nscorers = {\n 'r2': 'r2',\n 'nmsle': 'neg_mean_squared_log_error',\n 'nmse': 'neg_mean_squared_error',\n 'mae': 'neg_mean_absolute_error'\n}\n# To be used within GridSearch \ninner_cv = KFold(n_splits=4, shuffle=True, random_state=None)\n\n# To be used in outer CV \nouter_cv = KFold(n_splits=4, shuffle=True, random_state=None) \n\n#inner loop KFold example:\ngsc = GridSearchCV(\n estimator=ENet,\n param_grid=param_grid,\n scoring=scorers, # 'roc_auc', # or 'r2', etc -> can output several scores, but only refit to one. Others are just calculated\n #scoring='neg_mean_squared_error', # or look here for other choices \n # https://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter\n #cv=5,\n cv=inner_cv, # this will use KFold splitting to change train/test/validation datasets randomly\n verbose=0,\n return_train_score=True, # keep the other scores\n refit='nmse' # use this one for optimizing\n)\n\ngrid_result = runGSCV(5, train, y_train)\n\nrd = result_details(grid_result,'mean_test_nmse',100)\nrd[['max_iter','l1_ratio','mean_test_nmse','std_test_nmse','mean_test_mae','std_test_mae','mean_test_r2','std_test_r2']]#.sort_values(by=['n_estimators','mean_test_nmse'])\n","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:07.093106Z","iopub.execute_input":"2022-09-19T01:09:07.093446Z","iopub.status.idle":"2022-09-19T01:09:23.661704Z","shell.execute_reply.started":"2022-09-19T01:09:07.093412Z","shell.execute_reply":"2022-09-19T01:09:23.660531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rd = result_details(grid_result,'mean_test_nmse',100)\nrd[['max_iter','l1_ratio','mean_test_nmse','std_test_nmse','mean_test_mae','std_test_mae','mean_test_r2','std_test_r2']].sort_values(by=['max_iter','l1_ratio'])\n","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:23.666303Z","iopub.execute_input":"2022-09-19T01:09:23.671030Z","iopub.status.idle":"2022-09-19T01:09:23.731625Z","shell.execute_reply.started":"2022-09-19T01:09:23.670966Z","shell.execute_reply":"2022-09-19T01:09:23.730617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Linear Regression Model
","metadata":{}},{"cell_type":"code","source":"# initialize the algorithm for the GridSearchCV function\nlr1 = LinearRegression()\ntuningLR = 0 # takes 2 minutes to complete\n\nif (tuningLR == 1):\n # use this when tuning\n param_grid={\n 'fit_intercept':[True,False], \n 'normalize':[True,False]\n }\n\nelse:\n # use this when not tuning\n param_grid={\n 'fit_intercept':[False], \n 'normalize':[False]\n }\n\nscorers = {\n 'r2': 'r2',\n #'nmsle': 'neg_mean_squared_log_error',\n 'nmse': 'neg_mean_squared_error',\n 'mae': 'neg_mean_absolute_error'\n}\n# To be used within GridSearch \ninner_cv = KFold(n_splits=4, shuffle=True, random_state=None)\n\n# To be used in outer CV \nouter_cv = KFold(n_splits=4, shuffle=True, random_state=None) \n\n#inner loop KFold example:\ngsc = GridSearchCV(\n estimator=lr1,\n param_grid=param_grid,\n scoring=scorers, # 'roc_auc', # or 'r2', etc -> can output several scores, but only refit to one. Others are just calculated\n #scoring='neg_mean_squared_error', # or look here for other choices \n # https://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter\n #cv=5,\n cv=inner_cv, # this will use KFold splitting to change train/test/validation datasets randomly\n verbose=0,\n return_train_score=True, # keep the other scores\n refit='nmse' # use this one for optimizing\n)\n\ngrid_result = gsc.fit(train, y_train) \n#grid_result = runGSCV(2, train, y_train)\n\nrd = result_details(grid_result,'mean_test_nmse',100)\nrd[['fit_intercept','normalize','mean_test_nmse','std_test_nmse','mean_test_mae','std_test_mae','mean_test_r2','std_test_r2']]#.sort_values(by=['n_estimators','mean_test_nmse'])\n","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:23.735461Z","iopub.execute_input":"2022-09-19T01:09:23.735973Z","iopub.status.idle":"2022-09-19T01:09:24.510015Z","shell.execute_reply.started":"2022-09-19T01:09:23.735940Z","shell.execute_reply":"2022-09-19T01:09:24.508771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"polynomial_features = PolynomialFeatures(degree=2, include_bias=False)\nlr1 = LinearRegression(fit_intercept=True,normalize=False) # defaults fit_intercept=True, normalize=False\nfrom sklearn.feature_selection import f_regression, f_classif\n#lr_poly = Pipeline([(\"polynomial_features\", polynomial_features), (\"linear_regression\", LinearRegression(fit_intercept=True,normalize=False))])\n\n# using PCA\nlr_poly = Pipeline([(\"polynomial_features\", polynomial_features), ('reduce_dim', PCA(n_components=360)), (\"linear_regression\", LinearRegression())])\n\ntrans = PolynomialFeatures(degree=2)\ndata = trans.fit_transform(train)\nprint(data)\nprint(data.shape)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:24.556462Z","iopub.execute_input":"2022-09-19T01:09:24.564885Z","iopub.status.idle":"2022-09-19T01:09:24.801672Z","shell.execute_reply.started":"2022-09-19T01:09:24.564833Z","shell.execute_reply":"2022-09-19T01:09:24.800523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Ridge Regression Model
","metadata":{}},{"cell_type":"code","source":"tune_kr = 1\nif (tune_kr == 1):\n # initialize the algorithm for the GridSearchCV function\n KRR = KernelRidge()\n tuningKRR = 0 # this took 40 mins, 20 per iteration\n\n if (tuningKRR == 1):\n # use this when tuning\n param_grid={\n 'alpha':[2.2,2.4], \n 'kernel':['polynomial'], #for entire list see: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.pairwise.kernel_metrics.html#sklearn.metrics.pairwise.kernel_metrics\n 'gamma':[0.0001,0.001,0.01,0.1],\n 'degree':[1,2,3,4,5,6], \n 'coef0':[0.1,0.3,0.5,1.0,2.0]\n }\n\n else:\n # use this when not tuning\n # nmse: Best mae 583416973.611280 using {'alpha': 2.2, 'coef0': 0.5, 'degree': 5, 'gamma': 0.001, 'kernel': 'polynomial'}\n # mae: Best mae 15805.764347 using {'alpha': 2.0, 'coef0': 0.1, 'degree': 5, 'gamma': 0.001, 'kernel': 'polynomial'}\n param_grid={\n 'alpha':[2.2], \n 'kernel':['polynomial'], # 'linear', 'rbf'\n 'gamma':[0.001],\n 'degree':[4], \n 'coef0':[1.0]\n }\n scorers = {\n 'r2': 'r2',\n 'nmsle': 'neg_mean_squared_log_error',\n 'nmse': 'neg_mean_squared_error',\n 'mae': 'neg_mean_absolute_error'\n }\n # To be used within GridSearch \n inner_cv = KFold(n_splits=4, shuffle=True, random_state=None)\n # To be used in outer CV \n outer_cv = KFold(n_splits=4, shuffle=True, random_state=None) \n\n #inner loop KFold example:\n gsc = GridSearchCV(\n estimator=KRR,\n param_grid=param_grid,\n scoring=scorers, # 'roc_auc', # or 'r2', etc -> can output several scores, but only refit to one. Others are just calculated\n #scoring='neg_mean_squared_error', # or look here for other choices \n # https://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter\n #cv=5,\n cv=inner_cv, # this will use KFold splitting to change train/test/validation datasets randomly\n verbose=0,\n return_train_score=True, # keep the other scores\n refit='nmse' # use this one for optimizing\n )\n\n grid_result = runGSCV(2, train, y_train)\n\n rd = result_details(grid_result,'mean_test_nmse',100)\n rd[['alpha','degree','mean_test_nmse','std_test_nmse','mean_test_mae','std_test_mae','mean_test_r2','std_test_r2']]#.sort_values(by=['n_estimators','mean_test_nmse'])\n","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:24.805959Z","iopub.execute_input":"2022-09-19T01:09:24.808267Z","iopub.status.idle":"2022-09-19T01:09:36.375800Z","shell.execute_reply.started":"2022-09-19T01:09:24.808228Z","shell.execute_reply":"2022-09-19T01:09:36.374229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"krr = {'alpha': 2.2, 'coef0': 0.5, 'degree': 5, 'gamma': 0.001, 'kernel': 'polynomial'}\nKRR = KernelRidge(**krr)\n#KRR = KernelRidge(alpha=2.2, coef0=0.5, degree=5, gamma=0.001, kernel='polynomial')\n\nif (tune_kr == 1):\n KRR_new = KernelRidge(**grid_result.best_params_)\nelse:\n krr_new = {'alpha': 2.3, 'coef0': 1.0, 'degree': 4, 'gamma': 0.001, 'kernel': 'polynomial'}\n KRR_new = KernelRidge(**krr)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:36.382749Z","iopub.execute_input":"2022-09-19T01:09:36.387220Z","iopub.status.idle":"2022-09-19T01:09:36.401805Z","shell.execute_reply.started":"2022-09-19T01:09:36.387166Z","shell.execute_reply":"2022-09-19T01:09:36.400000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Random Forest Regression Model
","metadata":{}},{"cell_type":"code","source":"# initialize the algorithm for the GridSearchCV function\nrf = RandomForestRegressor()\ntuningRF = 0 # this took 2 hours last time, 1 hour per iteration\n\nif (tuningRF == 1):\n # use this when tuning\n param_grid={\n 'max_depth':[3,4,5],\n 'max_features':[None,'sqrt','log2'], \n # 'max_features': range(50,401,50),\n # 'max_features': [50,100], # can be list or range or other\n 'n_estimators':range(25,100,25), \n #'class_weight':[None,'balanced'], \n 'min_samples_leaf':range(5,15,5), \n 'min_samples_split':range(10,30,10), \n 'criterion':['mse', 'mae'] \n }\n\nelse:\n # use this when not tuning\n param_grid={\n 'max_depth':[5],\n 'max_features':[None], # max_features is None is default and works here, removing 'sqrt','log2'\n # 'max_features': range(50,401,50),\n # 'max_features': [50,100], # can be list or range or other\n 'n_estimators': [50], # number of trees selecting 100, removing range(50,126,25)\n #'class_weight':[None], # None was selected, removing 'balanced'\n 'min_samples_leaf': [5], #selecting 10, removing range 10,40,10)\n 'min_samples_split': [10], # selecting 20, removing range(20,80,10),\n 'criterion':['mse'] # remove gini as it is never selected\n }\n\nscorers = {\n 'r2': 'r2',\n 'nmsle': 'neg_mean_squared_log_error',\n 'nmse': 'neg_mean_squared_error',\n 'mae': 'neg_mean_absolute_error'\n}\n# To be used within GridSearch \ninner_cv = KFold(n_splits=4, shuffle=True, random_state=None)\n\n# To be used in outer CV \nouter_cv = KFold(n_splits=4, shuffle=True, random_state=None) \n\n#inner loop KFold example:\ngsc = GridSearchCV(\n estimator=rf,\n param_grid=param_grid,\n scoring=scorers, # 'roc_auc', # or 'r2', etc -> can output several scores, but only refit to one. Others are just calculated\n #scoring='neg_mean_squared_error', # or look here for other choices \n # https://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter\n #cv=5,\n cv=inner_cv, # this will use KFold splitting to change train/test/validation datasets randomly\n verbose=0,\n return_train_score=True, # keep the other scores\n refit='nmse' # use this one for optimizing\n)\n\ngrid_result = runGSCV(2, train, y_train)\n\nrd = result_details(grid_result,'mean_test_nmse',100)\nrd[['n_estimators','mean_test_nmse','std_test_nmse','mean_test_mae','std_test_mae','mean_test_r2','std_test_r2']]#.sort_values(by=['n_estimators','mean_test_nmse'])\n","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:36.410586Z","iopub.execute_input":"2022-09-19T01:09:36.414342Z","iopub.status.idle":"2022-09-19T01:09:55.592240Z","shell.execute_reply.started":"2022-09-19T01:09:36.414290Z","shell.execute_reply":"2022-09-19T01:09:55.590956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bps=grid_result.best_params_\n#bps['reg_alpha']=bps.pop(['alpha'])\nRF = make_pipeline(StandardScaler(), RandomForestRegressor(**bps)) # better than default, but still not good\nRF_new = make_pipeline(RobustScaler(), RandomForestRegressor(**bps)) # better than default, but still not good","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:55.594047Z","iopub.execute_input":"2022-09-19T01:09:55.594844Z","iopub.status.idle":"2022-09-19T01:09:55.600586Z","shell.execute_reply.started":"2022-09-19T01:09:55.594806Z","shell.execute_reply":"2022-09-19T01:09:55.599726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Optimize GBoost: \", datetime.datetime.now())","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:55.601996Z","iopub.execute_input":"2022-09-19T01:09:55.602348Z","iopub.status.idle":"2022-09-19T01:09:55.616990Z","shell.execute_reply.started":"2022-09-19T01:09:55.602315Z","shell.execute_reply":"2022-09-19T01:09:55.615980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Gradient Boosting Regression Model
","metadata":{}},{"cell_type":"code","source":"GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,\n max_depth=4, max_features='sqrt',\n min_samples_leaf=15, min_samples_split=10, \n loss='huber', random_state=5) # was 5\n# learning_ratefloat, default=0.1\n# learning rate shrinks the contribution of each tree by learning_rate. There is a trade-off between learning_rate and n_estimators.\ntuning_gb = 0\nif (tuning_gb == 1):\n # initialize the algorithm for the GridSearchCV function\n GBoost_new = GradientBoostingRegressor()\n tuningGB = 1\n if (tuningGB == 1):\n # use this when tuning\n param_grid={\n #'loss':['ls','lad','huber','quantile'],\n 'loss':['huber'], # done\n 'learning_rate':[0.05],\n 'n_estimators':[3000], # done\n 'subsample':[1.0],\n 'criterion':['friedman_mse'], # done\n 'min_samples_split':[10],\n 'min_samples_leaf':[15],\n 'min_weight_fraction_leaf':[0.0],\n 'max_depth':[2,3,4], # done\n 'min_impurity_decrease':[0.0],\n 'min_impurity_split':[None],\n 'init':[None],\n 'random_state':[None],\n 'max_features':[None,'auto','sqrt','log2'],\n 'max_features':['sqrt'], # done\n 'alpha':[0.60], # done\n 'verbose':[0],\n 'max_leaf_nodes':[None],\n 'warm_start':[False],\n 'presort':['deprecated'],\n 'validation_fraction':[0.1],\n 'n_iter_no_change':[None],\n 'tol':[0.0001],\n 'ccp_alpha':[0.0],\n 'random_state':[5,20,42]\n }\n else:\n # use this when not tuning\n param_grid={\n 'loss':['huber'], \n 'learning_rate':[0.05],\n 'n_estimators':[3000], \n 'subsample':[1.0],\n 'criterion':['friedman_mse'], \n 'min_samples_split':[10],\n 'min_samples_leaf':[15],\n 'min_weight_fraction_leaf':[0.0],\n 'max_depth':[2], \n 'min_impurity_decrease':[0.0],\n 'min_impurity_split':[None],\n 'init':[None],\n 'random_state':[None],\n 'max_features':['sqrt'], \n 'alpha':[0.60], \n 'verbose':[0],\n 'max_leaf_nodes':[None],\n 'warm_start':[False],\n 'presort':['deprecated'],\n 'validation_fraction':[0.1],\n 'n_iter_no_change':[None],\n 'tol':[0.0001],\n 'ccp_alpha':[0.0],\n 'random_state':[5]\n }\n scorers = {\n 'r2': 'r2',\n 'nmsle': 'neg_mean_squared_log_error',\n 'nmse': 'neg_mean_squared_error',\n 'mae': 'neg_mean_absolute_error'\n }\n # To be used within GridSearch \n inner_cv = KFold(n_splits=4, shuffle=True, random_state=None)\n # To be used in outer CV \n outer_cv = KFold(n_splits=4, shuffle=True, random_state=None) \n\n #inner loop KFold example:\n gsc = GridSearchCV(\n estimator=GBoost_new,\n param_grid=param_grid,\n scoring=scorers, # 'roc_auc', # or 'r2', etc -> can output several scores, but only refit to one. Others are just calculated\n cv=inner_cv, # this will use KFold splitting to change train/test/validation datasets randomly\n verbose=0,\n return_train_score=True, # keep the other scores\n refit='nmse' # use this one for optimizing\n )\n\n grid_result = runGSCV(2, subtrain, y_subtrain)\n\nrd = result_details(grid_result,'mean_test_nmse',100)\nrd[['criterion','max_depth','mean_test_nmse','std_test_nmse','mean_test_mae','std_test_mae','mean_test_r2','std_test_r2']]#.sort_values(by=['n_estimators','mean_test_nmse'])\n","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:55.618952Z","iopub.execute_input":"2022-09-19T01:09:55.619536Z","iopub.status.idle":"2022-09-19T01:09:55.651767Z","shell.execute_reply.started":"2022-09-19T01:09:55.619385Z","shell.execute_reply":"2022-09-19T01:09:55.650800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (tuning_gb == 1):\n GBoost_new = GradientBoostingRegressor(**grid_result.best_params_)\nelse:\n gbr = {'alpha': 0.6, 'ccp_alpha': 0.0, 'criterion': 'friedman_mse', 'init': None, 'learning_rate': 0.05, 'loss': 'huber', 'max_depth': 3, 'max_features': 'sqrt', 'max_leaf_nodes': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 15, 'min_samples_split': 10, 'min_weight_fraction_leaf': 0.0, 'n_estimators': 3000, 'n_iter_no_change': None, 'random_state': 5, 'subsample': 1.0, 'tol': 0.0001, 'validation_fraction': 0.1, 'verbose': 0, 'warm_start': False}\n GBoost_new = GradientBoostingRegressor(**gbr)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:55.653961Z","iopub.execute_input":"2022-09-19T01:09:55.654537Z","iopub.status.idle":"2022-09-19T01:09:55.664718Z","shell.execute_reply.started":"2022-09-19T01:09:55.654480Z","shell.execute_reply":"2022-09-19T01:09:55.663377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (tuning_gb == 1):\n #TEMP\n model_results = [] \n models = [GBoost, GBoost_new]\n\n for model in models:\n #print(model)\n with MyTimer(): \n scores = calc_all_scores(model,5,5)\n model_results.append([model, (-scores['test_mae']).mean(), (np.sqrt(-scores['test_nmse'])).mean(), scores['test_r2'].mean()])\n\n df_mr = pd.DataFrame(model_results, columns = ['model','mae','rmsle','r2'])\n df_mr.sort_values(by=['rmsle'])","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:55.666538Z","iopub.execute_input":"2022-09-19T01:09:55.667311Z","iopub.status.idle":"2022-09-19T01:09:55.677161Z","shell.execute_reply.started":"2022-09-19T01:09:55.667266Z","shell.execute_reply":"2022-09-19T01:09:55.676537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Testng - random test cases","metadata":{}},{"cell_type":"code","source":"if (tuning_gb == 1):\n GBoost.fit(subtrain.values, y_subtrain)\n gboost_train_pred = inv_boxcox1p(GBoost.predict(subtrain.values), lam_l)\n gboost_val_pred = inv_boxcox1p(GBoost.predict(val.values), lam_l)\n print('GBoost')\n print('train results')\n print(mae(y_subtrain, gboost_train_pred))\n print(rmsle(y_subtrain, gboost_train_pred))\n print('test results')\n print(mae(y_val, gboost_val_pred))\n print(rmsle(y_val, gboost_val_pred))\n\n GBoost_new.fit(subtrain.values, y_subtrain)\n gboost_train_pred = inv_boxcox1p(GBoost_new.predict(subtrain.values), lam_l)\n gboost_val_pred = inv_boxcox1p(GBoost_new.predict(val.values), lam_l)\n print('GBoost_new')\n print('train results')\n print(mae(y_subtrain, gboost_train_pred))\n print(rmsle(y_subtrain, gboost_train_pred))\n print('test results')\n print(mae(y_val, gboost_val_pred))\n print(rmsle(y_val, gboost_val_pred))","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:55.678406Z","iopub.execute_input":"2022-09-19T01:09:55.679009Z","iopub.status.idle":"2022-09-19T01:09:55.687768Z","shell.execute_reply.started":"2022-09-19T01:09:55.678975Z","shell.execute_reply":"2022-09-19T01:09:55.687076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (tuning_gb == 1):\n GBoost_new.fit(train, y_train)\n gboost_pred = inv_boxcox1p(GBoost_new.predict(test), lam_l)\n gboost_pred\n sub = pd.DataFrame()\n #sub['Id'] = test['Id']\n sub['Id'] = test_ID\n sub['SalePrice'] = gboost_pred\n sub.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:55.688944Z","iopub.execute_input":"2022-09-19T01:09:55.689512Z","iopub.status.idle":"2022-09-19T01:09:55.697185Z","shell.execute_reply.started":"2022-09-19T01:09:55.689478Z","shell.execute_reply":"2022-09-19T01:09:55.696603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (tuning_gb == 1):\n GBoost.fit(train, y_train)\n gboost_pred = inv_boxcox1p(GBoost.predict(test), lam_l)\n gboost_pred\n sub = pd.DataFrame()\n #sub['Id'] = test['Id']\n sub['Id'] = test_ID\n sub['SalePrice'] = gboost_pred\n sub.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:55.698335Z","iopub.execute_input":"2022-09-19T01:09:55.698919Z","iopub.status.idle":"2022-09-19T01:09:55.712285Z","shell.execute_reply.started":"2022-09-19T01:09:55.698884Z","shell.execute_reply":"2022-09-19T01:09:55.710890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (tuning_gb == 1):\n\n model_results = [] # model flow, mae, rmsle\n models = [GBoost, GBoost_new]\n\n for model in models:\n #print(model)\n with MyTimer(): \n scores = calc_all_scores(model,10,10)\n #print(\"------------------------------------------\")\n model_results.append([model, (-scores['test_mae']).mean(), (np.sqrt(-scores['test_nmse'])).mean(), scores['test_r2'].mean()])\n\n df_mr = pd.DataFrame(model_results, columns = ['model','mae','rmsle','r2'])\n df_mr.sort_values(by=['rmsle'])","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:55.714422Z","iopub.execute_input":"2022-09-19T01:09:55.714890Z","iopub.status.idle":"2022-09-19T01:09:55.725750Z","shell.execute_reply.started":"2022-09-19T01:09:55.714848Z","shell.execute_reply":"2022-09-19T01:09:55.724792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Extreame Gradient Boosting (XGBoost) Regression Model
","metadata":{}},{"cell_type":"code","source":"tuning_xgb = 0\nif (tuning_xgb == 1):\n # initialize the algorithm for the GridSearchCV function# initialize the algorithm for the GridSearchCV function\n xgb1 = xgb.XGBRegressor()\n tuningXGB = 1 # this took 2 hours last time, 1 hour per iteration\n\n if (tuningXGB == 1):\n # use this when tuning\n param_grid={\n 'colsample_bytree':[0.4603],\n 'gamma':[0.0468], # done - all values almost identical results\n 'colsample_bylevel':[0.3], # done - all give same result\n 'objective':['reg:squarederror'], # done - Default:'reg:squarederror', None, reg:pseudohubererror, reg:squaredlogerror, reg:gamma\n 'booster':['gbtree'], # done - Default: 'gbtree', 'gblinear' or 'dart'\n 'learning_rate':[0.04], # done\n 'max_depth':[3], # - done\n 'importance_type':['gain'], # done - all give same value, Default:'gain', 'weight', 'cover', 'total_gain' or 'total_cover'\n 'min_child_weight':[1.7817], # done - no difference with several values\n 'n_estimators':[1000], # done\n 'reg_alpha':[0.4], # done\n 'reg_lambda':[0.8571], # done\n 'subsample':[0.5], # done\n 'silent':[1],\n 'random_state':[35],\n 'scale_pos_weight':[1],\n 'eval_metric':['rmse'], # done - all options have same results Default:rmse for regression rmse, mae, rmsle, logloss, cox-nloglik\n #'nthread ':[-1],\n 'verbosity':[0]\n }\n\n else:\n # use this when not tuning\n param_grid={\n 'colsample_bytree':[0.4603],\n 'gamma':[0.0468],\n 'colsample_bylevel':[0.3],\n 'objective':['reg:squarederror'], # 'binary:logistic', 'reg:squarederror', 'rank:pairwise', None\n 'booster':['gbtree'], # 'gbtree', 'gblinear' or 'dart'\n 'learning_rate':[0.04],\n 'max_depth':[3],\n 'importance_type':['gain'], # 'gain', 'weight', 'cover', 'total_gain' or 'total_cover'\n 'min_child_weight':[1.7817],\n 'n_estimators':[1000],\n 'reg_alpha':[0.4],\n 'reg_lambda':[0.8571],\n 'subsample':[0.5],\n 'silent':[1],\n 'random_state':[35],\n 'scale_pos_weight':[1],\n 'eval_metric':['rmse'],\n #'nthread ':[-1],\n 'nthread ':[-1],\n 'verbosity':[0]\n }\n\n scorers = {\n 'r2': 'r2',\n 'nmsle': 'neg_mean_squared_log_error',\n 'nmse': 'neg_mean_squared_error',\n 'mae': 'neg_mean_absolute_error'\n }\n # To be used within GridSearch \n inner_cv = KFold(n_splits=4, shuffle=True, random_state=None)\n\n # To be used in outer CV \n outer_cv = KFold(n_splits=4, shuffle=True, random_state=None) \n\n #inner loop KFold example:\n gsc = GridSearchCV(\n estimator=xgb1,\n param_grid=param_grid,\n scoring=scorers, # 'roc_auc', # or 'r2', etc -> can output several scores, but only refit to one. Others are just calculated\n #scoring='neg_mean_squared_error', # or look here for other choices \n # https://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter\n #cv=5,\n cv=inner_cv, # this will use KFold splitting to change train/test/validation datasets randomly\n verbose=0,\n return_train_score=True, # keep the other scores\n refit='nmse' # use this one for optimizing\n )\n\n grid_result = runGSCV(2, train, y_train)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:55.727162Z","iopub.execute_input":"2022-09-19T01:09:55.727765Z","iopub.status.idle":"2022-09-19T01:09:55.742212Z","shell.execute_reply.started":"2022-09-19T01:09:55.727728Z","shell.execute_reply":"2022-09-19T01:09:55.741243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (tuning_xgb == 1):\n rd = result_details(grid_result,'mean_test_nmse',100)\n\n rd[['random_state','eval_metric','mean_test_nmse','std_test_nmse','mean_test_mae','std_test_mae','mean_test_r2','std_test_r2']].sort_values(by=['random_state','mean_test_nmse'])\n","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:55.745851Z","iopub.execute_input":"2022-09-19T01:09:55.746155Z","iopub.status.idle":"2022-09-19T01:09:55.755750Z","shell.execute_reply.started":"2022-09-19T01:09:55.746131Z","shell.execute_reply":"2022-09-19T01:09:55.754878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_xgb = xgb.XGBRegressor(booster='gbtree',colsample_bytree=0.4603, gamma=0.0468, # colsample_bylevel, objective, booster (gbtree, gblinear or dart.) # Default: 'gbtree'\n learning_rate=0.05, max_depth=3, # importance_type (βgainβ, βweightβ, βcoverβ, βtotal_gainβ or βtotal_coverβ.)\n min_child_weight=1.7817, n_estimators=2200,\n reg_alpha=0.4640, reg_lambda=0.8571,\n subsample=0.5213, silent=1,\n random_state=35) # was random_state=7, cannot set to None \n\nif (tuning_xgb == 1):\n import xgboost as xgb\n for i in [2,5,20,42,99]:\n print('random_state =',i)\n\n model_xgb_new = xgb.XGBRegressor(booster='gbtree',colsample_bytree=0.4603, gamma=0.0468, # colsample_bylevel, objective, booster (gbtree, gblinear or dart.) # Default: 'gbtree'\n learning_rate=0.04, max_depth=3, # importance_type (βgainβ, βweightβ, βcoverβ, βtotal_gainβ or βtotal_coverβ.)\n min_child_weight=1.7817, n_estimators=1000,\n reg_alpha=0.4, reg_lambda=0.8571,\n subsample=0.45, silent=1,\n random_state=i) # was random_state=7, cannot set to None \n\n model_xgb_new2 = xgb.XGBRegressor(booster='gbtree',colsample_bytree=0.4603, gamma=0.0468, # colsample_bylevel, objective, booster (gbtree, gblinear or dart.) # Default: 'gbtree'\n learning_rate=0.04, max_depth=3, # importance_type (βgainβ, βweightβ, βcoverβ, βtotal_gainβ or βtotal_coverβ.)\n min_child_weight=1.7817, n_estimators=1000,\n reg_alpha=0.4, reg_lambda=0.8571,\n subsample=0.5, silent=1,\n random_state=i) # was random_state=7, cannot set to None \n\n model_xgb_new3 = xgb.XGBRegressor(booster='gbtree',colsample_bytree=0.4603, gamma=0.0468, # colsample_bylevel, objective, booster (gbtree, gblinear or dart.) # Default: 'gbtree'\n learning_rate=0.04, max_depth=3, # importance_type (βgainβ, βweightβ, βcoverβ, βtotal_gainβ or βtotal_coverβ.)\n min_child_weight=1.7817, n_estimators=1000,\n reg_alpha=0.4, reg_lambda=0.8571,\n subsample=0.5213, silent=1,\n random_state=i) # was random_state=7, cannot set to None\n\n model_results = [] # model flow, mae, rmsle\n models = [model_xgb, model_xgb_new, model_xgb_new2, model_xgb_new3]\n\n for model in models:\n #print(model)\n with MyTimer(): \n scores = calc_all_scores(model,5,5)\n #print(\"------------------------------------------\")\n model_results.append([model, (-scores['test_mae']).mean(), (np.sqrt(-scores['test_nmse'])).mean(), scores['test_r2'].mean()])\n\n df_mr = pd.DataFrame(model_results, columns = ['model','mae','rmsle','r2'])\n print(df_mr.sort_values(by=['rmsle']))\nelse:\n model_xgb_new = xgb.XGBRegressor(**grid_result.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:55.757163Z","iopub.execute_input":"2022-09-19T01:09:55.757551Z","iopub.status.idle":"2022-09-19T01:09:55.772439Z","shell.execute_reply.started":"2022-09-19T01:09:55.757515Z","shell.execute_reply":"2022-09-19T01:09:55.771139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_metrics = 0\nif (show_metrics == 1):\n import graphviz\n model_xgb.fit(train, y_train, verbose=False) # eval_set=[(X_test, y_test)]\n xgb.plot_importance(model_xgb)\n xgb.to_graphviz(model_xgb, num_trees=20)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:55.774534Z","iopub.execute_input":"2022-09-19T01:09:55.775014Z","iopub.status.idle":"2022-09-19T01:09:55.786545Z","shell.execute_reply.started":"2022-09-19T01:09:55.774975Z","shell.execute_reply":"2022-09-19T01:09:55.785497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Light Gradient Boosting (LGBoost) Regression Model
","metadata":{}},{"cell_type":"code","source":"tuning_lgb = 0\nif (tuning_lgb == 1):\n lgb1 = lgb.LGBMRegressor()\n tuningLGB = 0\n\n if (tuningLGB == 1):\n # use this when tuning\n param_grid={\n 'objective':['regression'], # - only one option for regression\n 'boosting_type':['gbdt'], # - done gbdt dart goss rf\n 'num_leaves':[5,6], # - done\n 'learning_rate':[0.05], # - done\n 'n_estimators':[650,750], # - done\n 'max_bin':[45,55], # - done\n 'bagging_fraction':[0.85], # - done\n 'bagging_freq':[5], # - done\n 'feature_fraction':[0.2319], # - done\n 'feature_fraction_seed':[9], \n 'bagging_seed':[9],\n 'min_data_in_leaf':[9], # - done\n 'min_sum_hessian_in_leaf':[11], # - done\n 'max_depth':[-1], # - -1 means no limit\n 'subsample_for_bin':[500,1000], # - done\n 'class_weight':[None],\n 'min_split_gain':[0.0],\n 'min_child_weight':[0.001],\n 'min_child_samples':[5], # - done\n 'subsample':[1.0],\n 'subsample_freq':[0],\n 'colsample_bytree':[1.0],\n 'reg_alpha':[0.0], # - l1 regularization done\n 'reg_lambda':[0.0], # - L2 regularization done\n 'random_state':[1],\n 'importance_type':['split'] # - done\n }\n else:\n # use this when not tuning\n param_grid={\n 'objective':['regression'], # - only one option for regression\n 'boosting_type':['gbdt'], # - done gbdt dart goss rf\n 'num_leaves':[5], # - done, maybe 5 is okay too\n 'learning_rate':[0.05], # - done\n 'n_estimators':[650], # - done\n 'max_bin':[55], # - done\n 'bagging_fraction':[0.85], # - done\n 'bagging_freq':[5], # - done\n 'feature_fraction':[0.2319], # - done\n 'feature_fraction_seed':[9], \n 'bagging_seed':[9],\n 'min_data_in_leaf':[9], # - done\n 'min_sum_hessian_in_leaf':[11], # - done\n 'max_depth':[-1], # - -1 means no limit\n 'subsample_for_bin':[1000], # - done\n 'class_weight':[None],\n 'min_split_gain':[0.0],\n 'min_child_weight':[0.001],\n 'min_child_samples':[5], # - done\n 'subsample':[1.0],\n 'subsample_freq':[0],\n 'colsample_bytree':[1.0],\n 'reg_alpha':[0.0], # - l1 regularization done\n 'reg_lambda':[0.0], # - L2 regularization done\n 'random_state':[1],\n 'importance_type':['split'] # - done\n }\n\n scorers = {\n 'r2': 'r2',\n 'nmsle': 'neg_mean_squared_log_error',\n 'nmse': 'neg_mean_squared_error',\n 'mae': 'neg_mean_absolute_error'\n }\n # To be used within GridSearch \n inner_cv = KFold(n_splits=4, shuffle=True, random_state=None)\n\n # To be used in outer CV \n outer_cv = KFold(n_splits=4, shuffle=True, random_state=None) \n\n #inner loop KFold example:\n gsc = GridSearchCV(\n estimator=lgb1,\n param_grid=param_grid,\n scoring=scorers, # 'roc_auc', # or 'r2', etc -> can output several scores, but only refit to one. Others are just calculated\n #scoring='neg_mean_squared_error', # or look here for other choices \n # https://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter\n #cv=5,\n cv=inner_cv, # this will use KFold splitting to change train/test/validation datasets randomly\n verbose=0,\n return_train_score=True, # keep the other scores\n refit='nmse' # use this one for optimizing\n )\n\n grid_result = runGSCV(2, train, y_train)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:55.789862Z","iopub.execute_input":"2022-09-19T01:09:55.790148Z","iopub.status.idle":"2022-09-19T01:09:55.805549Z","shell.execute_reply.started":"2022-09-19T01:09:55.790114Z","shell.execute_reply":"2022-09-19T01:09:55.804601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,\n learning_rate=0.05, n_estimators=720,\n max_bin = 55, bagging_fraction = 0.8,\n bagging_freq = 5, feature_fraction = 0.2319,\n feature_fraction_seed=9, bagging_seed=9, random_state=10,\n min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)\nif (tuning_lgb == 1):\n model_lgb_new = lgb.LGBMRegressor(**grid_result.best_params_)\nelse:\n lgbm = {'bagging_fraction': 0.85, 'bagging_freq': 5, 'bagging_seed': 9, 'boosting_type': 'gbdt', 'class_weight': None, 'colsample_bytree': 1.0, 'feature_fraction': 0.2319, 'feature_fraction_seed': 9, 'importance_type': 'split', 'learning_rate': 0.05, 'max_bin': 55, 'max_depth': -1, 'min_child_samples': 5, 'min_child_weight': 0.001, 'min_data_in_leaf': 9, 'min_split_gain': 0.0, 'min_sum_hessian_in_leaf': 11, 'n_estimators': 650, 'num_leaves': 5, 'objective': 'regression', 'random_state': None, 'reg_alpha': 0.0, 'reg_lambda': 0.0, 'subsample': 1.0, 'subsample_for_bin': 1000, 'subsample_freq': 0}\n model_lgb_new = lgb.LGBMRegressor(**lgbm)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:55.808616Z","iopub.execute_input":"2022-09-19T01:09:55.808867Z","iopub.status.idle":"2022-09-19T01:09:55.819542Z","shell.execute_reply.started":"2022-09-19T01:09:55.808844Z","shell.execute_reply":"2022-09-19T01:09:55.818717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Bayesian Ridge Regression Model
","metadata":{}},{"cell_type":"code","source":"# initialize the algorithm for the GridSearchCV function\nbr = BayesianRidge()\ntuningBR = 1 # this took 2 hours last time, 1 hour per iteration\n\nif (tuningBR == 1):\n # use this when tuning\n param_grid={\n 'n_iter':[50],\n 'tol':[0.001],\n 'alpha_1':[1e-06],\n 'alpha_2':[1e-05],\n 'lambda_1':[1e-05],\n 'lambda_2':[1e-06],\n 'alpha_init':[None],\n 'lambda_init':[None],\n 'compute_score':[True,False],\n 'fit_intercept':[True,False],\n 'normalize':[False,True],\n 'copy_X':[True],\n 'verbose':[False]\n }\n\nelse:\n # use this when not tuning\n param_grid={\n 'n_iter':[50],\n 'tol':[0.001],\n 'alpha_1':[1e-06],\n 'alpha_2':[1e-05],\n 'lambda_1':[1e-05],\n 'lambda_2':[1e-06],\n 'alpha_init':[None],\n 'lambda_init':[None],\n 'compute_score':[True,False],\n 'fit_intercept':[True,False],\n 'normalize':[False,True],\n 'copy_X':[True],\n 'verbose':[False]\n }\n\nscorers = {\n 'r2': 'r2',\n 'nmsle': 'neg_mean_squared_log_error',\n 'nmse': 'neg_mean_squared_error',\n 'mae': 'neg_mean_absolute_error'\n}\n# To be used within GridSearch \ninner_cv = KFold(n_splits=4, shuffle=True, random_state=None)\n\n# To be used in outer CV \nouter_cv = KFold(n_splits=4, shuffle=True, random_state=None) \n\n#inner loop KFold example:\ngsc = GridSearchCV(\n estimator=br,\n param_grid=param_grid,\n scoring=scorers, # 'roc_auc', # or 'r2', etc -> can output several scores, but only refit to one. Others are just calculated\n #scoring='neg_mean_squared_error', # or look here for other choices \n # https://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter\n #cv=5,\n cv=inner_cv, # this will use KFold splitting to change train/test/validation datasets randomly\n verbose=0,\n return_train_score=True, # keep the other scores\n refit='nmse' # use this one for optimizing\n)\n\ngrid_result = runGSCV(2, train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:09:55.821491Z","iopub.execute_input":"2022-09-19T01:09:55.821856Z","iopub.status.idle":"2022-09-19T01:10:36.890290Z","shell.execute_reply.started":"2022-09-19T01:09:55.821814Z","shell.execute_reply":"2022-09-19T01:10:36.889029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tuning_br = 0\nBR = BayesianRidge()\nif (tuning_br == 1):\n BR_new = BayesianRidge(**grid_result.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:10:36.896243Z","iopub.execute_input":"2022-09-19T01:10:36.896982Z","iopub.status.idle":"2022-09-19T01:10:36.905839Z","shell.execute_reply.started":"2022-09-19T01:10:36.896927Z","shell.execute_reply":"2022-09-19T01:10:36.904368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rd = result_details(grid_result,'alpha_1',100)\nrd[['alpha_1','alpha_2','mean_test_nmse','std_test_nmse','mean_test_mae','std_test_mae','mean_test_r2','std_test_r2']]","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:10:36.907385Z","iopub.execute_input":"2022-09-19T01:10:36.907949Z","iopub.status.idle":"2022-09-19T01:10:36.962779Z","shell.execute_reply.started":"2022-09-19T01:10:36.907908Z","shell.execute_reply":"2022-09-19T01:10:36.961739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (tuning_br == 1):\n model_results = [] # model flow, mae, rmsle\n models = [BR, BR_new]\n\n for model in models:\n #print(model)\n with MyTimer(): \n scores = calc_all_scores(model,10,10)\n #print(\"------------------------------------------\")\n model_results.append([model, (-scores['test_mae']).mean(), (np.sqrt(-scores['test_nmse'])).mean(), scores['test_r2'].mean()])\n\n df_mr = pd.DataFrame(model_results, columns = ['model','mae','rmsle','r2'])\n df_mr.sort_values(by=['rmsle'])","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:10:36.964034Z","iopub.execute_input":"2022-09-19T01:10:36.965654Z","iopub.status.idle":"2022-09-19T01:10:36.981587Z","shell.execute_reply.started":"2022-09-19T01:10:36.965616Z","shell.execute_reply":"2022-09-19T01:10:36.979954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# initialize the algorithm for the GridSearchCV function\nR = Ridge(alpha=1.0)\ntuningR = 1 # this took 2 hours last time, 1 hour per iteration\n\nif (tuningR == 1):\n # use this when tuning\n param_grid={\n 'alpha':[8], # done\n 'fit_intercept':[True], # done\n 'normalize':[False], # done\n 'copy_X':[True],\n 'max_iter':[None], # done - no difference\n 'tol':[0.001],\n 'solver':['auto'], # done - Default:βautoβ, βsvdβ, βcholeskyβ, βlsqrβ, βsparse_cgβ, βsagβ, βsagaβ\n 'random_state':[1,10,42,99,127]\n }\n\nelse:\n # use this when not tuning\n param_grid={\n 'alpha':[1.0],\n 'fit_intercept':[True],\n 'normalize':[False],\n 'copy_X':[True],\n 'max_iter':[None],\n 'tol':[0.001],\n 'solver':['auto'],\n 'random_state':[None]\n }\n\nscorers = {\n 'r2': 'r2',\n 'nmsle': 'neg_mean_squared_log_error',\n 'nmse': 'neg_mean_squared_error',\n 'mae': 'neg_mean_absolute_error'\n}\n# To be used within GridSearch \ninner_cv = KFold(n_splits=4, shuffle=True, random_state=None)\n\n# To be used in outer CV \nouter_cv = KFold(n_splits=4, shuffle=True, random_state=None) \n\n#inner loop KFold example:\ngsc = GridSearchCV(\n estimator=R,\n param_grid=param_grid,\n scoring=scorers, # 'roc_auc', # or 'r2', etc -> can output several scores, but only refit to one. Others are just calculated\n #scoring='neg_mean_squared_error', # or look here for other choices \n # https://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter\n #cv=5,\n cv=inner_cv, # this will use KFold splitting to change train/test/validation datasets randomly\n verbose=0,\n return_train_score=True, # keep the other scores\n refit='nmse' # use this one for optimizing\n)\n\ngrid_result = runGSCV(5, train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:10:36.983993Z","iopub.execute_input":"2022-09-19T01:10:36.985868Z","iopub.status.idle":"2022-09-19T01:11:05.373838Z","shell.execute_reply.started":"2022-09-19T01:10:36.985820Z","shell.execute_reply":"2022-09-19T01:11:05.372601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rd = result_details(grid_result,'mean_test_nmse',100)\nsummary = rd[['alpha','random_state','mean_test_nmse','std_test_nmse','mean_test_mae','std_test_mae','mean_test_r2','std_test_r2']].sort_values(by=['random_state','mean_test_nmse'])\n#summary.groupby(['fit_intercept'], as_index=False).agg({'mean_test_nmse': 'mean', 'mean_test_nmse': 'std', 'mean_test_mae': 'mean', 'mean_test_r2': 'mean'}).sort_values(by=['mean_test_mae'])\nsummary.groupby(['alpha'], as_index=False).agg({'mean_test_nmse': 'mean', 'mean_test_mae': 'mean', 'mean_test_r2': 'mean'}).sort_values(by=['mean_test_mae'])\n","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:11:05.379437Z","iopub.execute_input":"2022-09-19T01:11:05.382941Z","iopub.status.idle":"2022-09-19T01:11:05.451483Z","shell.execute_reply.started":"2022-09-19T01:11:05.382887Z","shell.execute_reply":"2022-09-19T01:11:05.450043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tuning_r = 1\nif (tuning_r == 1):\n for i in [2,5,20,42,99]:\n print('random_state =',i)\n\n r= {'alpha': 8, 'copy_X': True, 'fit_intercept': True, 'max_iter': None, 'normalize': False, 'solver': 'auto', 'tol': 0.001}\n #R_new = make_pipeline(RobustScaler(), Ridge(**r, random_state=i))\n R_new = Ridge(**r, random_state=i)\n\n model_results = [] # model flow, mae, rmsle\n models = [R, R_new]\n\n for model in models:\n #print(model)\n with MyTimer(): \n scores = calc_all_scores(model,5,5)\n #print(\"------------------------------------------\")\n model_results.append([model, (-scores['test_mae']).mean(), (np.sqrt(-scores['test_nmse'])).mean(), scores['test_r2'].mean()])\n\n df_mr = pd.DataFrame(model_results, columns = ['model','mae','rmsle','r2'])\n print(df_mr.sort_values(by=['rmsle']))\nelse:\n #R_new = make_pipeline(RobustScaler(), Ridge(**grid_result.best_params_))\n R_new = Ridge(**grid_result.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:11:05.452923Z","iopub.execute_input":"2022-09-19T01:11:05.453252Z","iopub.status.idle":"2022-09-19T01:11:08.522539Z","shell.execute_reply.started":"2022-09-19T01:11:05.453220Z","shell.execute_reply":"2022-09-19T01:11:08.520775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"CatBoost Regression Model
","metadata":{}},{"cell_type":"code","source":"#CBoost = CatBoostRegressor(logging_level='Silent',random_seed=0)\nCBoost = CatBoostRegressor(logging_level='Silent', random_state=0, depth=6, n_estimators=1000,eval_metric='RMSE',bagging_temperature=1,grow_policy='SymmetricTree',bootstrap_type='MVS') # l2_leaf_reg, learning_rate unknown\n\ntuning_cb = 0\nif (tuning_cb == 1):\n # initialize the algorithm for the GridSearchCV function\n cb = CatBoostRegressor()\n if (tuningCB == 1):\n # use this when tuning\n param_grid={\n 'nan_mode':['Min'],\n 'eval_metric':['RMSE'],\n 'iterations':[1000],\n 'sampling_frequency':['PerTree'],\n 'leaf_estimation_method':['Newton'],\n 'grow_policy':['SymmetricTree'],\n 'penalties_coefficient':[1],\n 'boosting_type':['Plain'],\n 'model_shrink_mode':['Constant'],\n 'feature_border_type':['GreedyLogSum'],\n #'bayesian_matrix_reg':[0.10000000149011612],\n 'l2_leaf_reg':[3],\n 'random_strength':[1],\n 'rsm':[1],\n 'boost_from_average':[True],\n 'model_size_reg':[0.5],\n 'subsample':[0.800000011920929],\n 'use_best_model':[False],\n 'random_seed':[0,2,15],\n 'depth':[6], # done\n 'border_count':[254],\n #'classes_count':[0],\n #'auto_class_weights':['None'],\n 'sparse_features_conflict_fraction':[0],\n 'leaf_estimation_backtracking':['AnyImprovement'],\n 'best_model_min_trees':[1],\n 'model_shrink_rate':[0],\n 'min_data_in_leaf':[1],\n 'loss_function':['RMSE'],\n 'learning_rate':[0.04174000024795532],\n 'score_function':['Cosine'],\n 'task_type':['CPU'],\n 'leaf_estimation_iterations':[1],\n 'bootstrap_type':['MVS'],\n 'max_leaves':[31],\n 'logging_level':['Silent']\n }\n\n else:\n # use this when not tuning\n param_grid={\n 'nan_mode':['Min'],\n 'eval_metric':['RMSE'],\n 'iterations':[4000],\n 'sampling_frequency':['PerTree'],\n 'leaf_estimation_method':['Newton'],\n 'grow_policy':['SymmetricTree'],\n 'penalties_coefficient':[1],\n 'boosting_type':['Plain'],\n 'model_shrink_mode':['Constant'],\n 'feature_border_type':['GreedyLogSum'],\n #'bayesian_matrix_reg':[0.10000000149011612],\n 'l2_leaf_reg':[3],\n 'random_strength':[1],\n 'rsm':[1],\n 'boost_from_average':[True],\n 'model_size_reg':[0.5],\n 'subsample':[0.800000011920929],\n 'use_best_model':[False],\n 'random_seed':[15],\n 'depth':[6],\n 'border_count':[254],\n #'classes_count':[0],\n #'auto_class_weights':['None'],\n 'sparse_features_conflict_fraction':[0],\n 'leaf_estimation_backtracking':['AnyImprovement'],\n 'best_model_min_trees':[1],\n 'model_shrink_rate':[0],\n 'min_data_in_leaf':[1],\n 'loss_function':['RMSE'],\n 'learning_rate':[0.04174000024795532],\n 'score_function':['Cosine'],\n 'task_type':['CPU'],\n 'leaf_estimation_iterations':[1],\n 'bootstrap_type':['MVS'],\n 'max_leaves':[31],\n 'logging_level':['Silent']\n }\n\n scorers = {\n 'r2': 'r2',\n 'nmsle': 'neg_mean_squared_log_error',\n 'nmse': 'neg_mean_squared_error',\n 'mae': 'neg_mean_absolute_error'\n }\n # To be used within GridSearch \n inner_cv = KFold(n_splits=4, shuffle=True, random_state=None)\n\n # To be used in outer CV \n outer_cv = KFold(n_splits=4, shuffle=True, random_state=None) \n\n #inner loop KFold example:\n gsc = GridSearchCV(\n estimator=cb,\n param_grid=param_grid,\n scoring=scorers, # 'roc_auc', # or 'r2', etc -> can output several scores, but only refit to one. Others are just calculated\n #scoring='neg_mean_squared_error', # or look here for other choices \n # https://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter\n #cv=5,\n cv=inner_cv, # this will use KFold splitting to change train/test/validation datasets randomly\n verbose=0,\n return_train_score=True, # keep the other scores\n refit='nmse'#, # use this one for optimizing\n #plot=True\n )\n grid_result = runGSCV(2, train, y_train)\n rd = result_details(grid_result,'depth',100)\n rd[['random_state','depth','mean_test_nmse','std_test_nmse','mean_test_mae','std_test_mae','mean_test_r2','std_test_r2']]","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:11:08.529215Z","iopub.execute_input":"2022-09-19T01:11:08.533845Z","iopub.status.idle":"2022-09-19T01:11:08.580558Z","shell.execute_reply.started":"2022-09-19T01:11:08.533792Z","shell.execute_reply":"2022-09-19T01:11:08.579182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nif (tuning_cb == 1):\n for i in [2,15,20,42,99]:\n print('random_state =', i)\n #CBoost = CatBoostRegressor(logging_level='Silent', random_state=i) # don't touch this variable\n #CBoost_def = CatBoostRegressor(logging_level='Silent', random_state=i, depth=6, n_estimators=1000,eval_metric='RMSE',bagging_temperature=1,grow_policy='SymmetricTree',bootstrap_type='MVS') # l2_leaf_reg, learning_rate unknown \n # 'max_leaves': 64, creates nan values, default of 31 is fine, or remove entirely...\n CBoost_new = CatBoostRegressor(**grid_result.best_params_) \n\n model_results = [] # model flow, mae, rmsle\n models = [CBoost, CBoost_new]\n\n for model in models:\n #print(model)\n with MyTimer(): \n scores = calc_all_scores(model,5,5)\n model_results.append([model.get_params(), (-scores['test_mae']).mean(), (np.sqrt(-scores['test_nmse'])).mean(), scores['test_r2'].mean()])\n\n df_mr = pd.DataFrame(model_results, columns = ['model','mae','rmsle','r2'])\n print(df_mr.sort_values(by=['rmsle']))\nelse:\n # was n_estimators=3500\n CBoost_new = CatBoostRegressor(logging_level='Silent', random_state=15, depth=5, l2_leaf_reg=1.0, n_estimators=1700,eval_metric='RMSE',learning_rate=0.025,random_strength=3.7,bagging_temperature=1.0,grow_policy='SymmetricTree',bootstrap_type='Bayesian')#,bayesian_matrix_reg=0.10000000149011612)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:11:08.586803Z","iopub.execute_input":"2022-09-19T01:11:08.589916Z","iopub.status.idle":"2022-09-19T01:11:08.608483Z","shell.execute_reply.started":"2022-09-19T01:11:08.589864Z","shell.execute_reply":"2022-09-19T01:11:08.607123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from copy import deepcopy\nimport pickle\nCBoost_new2 = deepcopy(CBoost_new)\nCBoost_new2.fit(train,y_train)\nwith open('CBoost_new.pkl', 'wb') as fid:\n pickle.dump(CBoost_new2, fid) \n# load it again\nwith open('CBoost_new.pkl', 'rb') as fid:\n CBoost_new2 = pickle.load(fid)\nCBoost_new2.predict(train[0:1])","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:11:08.615730Z","iopub.execute_input":"2022-09-19T01:11:08.619595Z","iopub.status.idle":"2022-09-19T01:11:16.591602Z","shell.execute_reply.started":"2022-09-19T01:11:08.619534Z","shell.execute_reply":"2022-09-19T01:11:16.590516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nCBoost_poly = Pipeline([(\"polynomial_features\", polynomial_features), (\"cboost_regression\", CBoost_new)])","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:11:16.593113Z","iopub.execute_input":"2022-09-19T01:11:16.593598Z","iopub.status.idle":"2022-09-19T01:11:16.598623Z","shell.execute_reply.started":"2022-09-19T01:11:16.593544Z","shell.execute_reply":"2022-09-19T01:11:16.597679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AB = AdaBoostRegressor()\nfrom sklearn.svm import SVR\nSVR = SVR()\nDT = DecisionTreeRegressor()\nKN = KNeighborsRegressor()\nB = BaggingRegressor()","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:11:16.600134Z","iopub.execute_input":"2022-09-19T01:11:16.600793Z","iopub.status.idle":"2022-09-19T01:11:16.609047Z","shell.execute_reply.started":"2022-09-19T01:11:16.600760Z","shell.execute_reply":"2022-09-19T01:11:16.608067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Scores Analysis
","metadata":{}},{"cell_type":"code","source":"if (tuning_gb == 1):\n model_results = [] # model flow, mae, rmsle\n models = [GBoost, GBoost_orig]#, GBoost] # model_lgb_op, lasso_ns, \n\n for model in models:\n #print(model)\n with MyTimer(): \n scores = calc_all_scores(model)\n #print(\"------------------------------------------\")\n model_results.append([model, (-scores['test_mae']).mean(), (np.sqrt(-scores['test_nmse'])).mean(), scores['test_r2'].mean()])\n\n df_mr = pd.DataFrame(model_results, columns = ['model','mae','rmsle','r2'])\n df_mr.sort_values(by=['rmsle'])","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:11:16.611038Z","iopub.execute_input":"2022-09-19T01:11:16.612995Z","iopub.status.idle":"2022-09-19T01:11:16.621084Z","shell.execute_reply.started":"2022-09-19T01:11:16.612969Z","shell.execute_reply":"2022-09-19T01:11:16.620038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"compare_models = 0\nif (compare_models == 1):\n model_results = [] # model flow, mae, rmsle\n # GBoost_new is better than GBoost, but has lower final score, think this may be overfitting\n # BR_new has same results, here, but better final score\n models = [lasso, lasso_new, model_lgb, ENet, ENet_new, KRR, GBoost, GBoost_new, model_xgb, BR, ET, ET_new, RF, RF_new, AB, SVR, DT, KN, B, R, R_new, CBoost, CBoost_new] # worse or same: BR_new, model_lgb_new, lasso_ns, model_xgb_new,\n\n for model in models:\n #print(model)\n with MyTimer(): \n scores = calc_all_scores(model)\n #print(\"------------------------------------------\")\n try:\n print(model)\n label = model\n except KeyError as err:\n print(\"KeyError error: {0}\".format(err))\n label = model.__class__()\n except Exception as e:\n print(e.message, e.args)\n label = model.__class__()\n finally:\n print(\"Continue\") \n print('label', label) \n #model_results.append([model, (-scores['test_mae']).mean(), (np.sqrt(-scores['test_nmse'])).mean(), scores['test_r2'].mean()])\n model_results.append([label, (-scores['test_mae']).mean(), (np.sqrt(-scores['test_nmse'])).mean(), scores['test_r2'].mean()])\n\n df_mr = pd.DataFrame(model_results, columns = ['model','mae','rmsle','r2'])\n df_mr.sort_values(by=['rmsle'])","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:11:16.622889Z","iopub.execute_input":"2022-09-19T01:11:16.623321Z","iopub.status.idle":"2022-09-19T01:11:16.638283Z","shell.execute_reply.started":"2022-09-19T01:11:16.623258Z","shell.execute_reply":"2022-09-19T01:11:16.637100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_mr.sort_values(by=['rmsle'])","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:11:16.640061Z","iopub.execute_input":"2022-09-19T01:11:16.640802Z","iopub.status.idle":"2022-09-19T01:11:16.656866Z","shell.execute_reply.started":"2022-09-19T01:11:16.640760Z","shell.execute_reply":"2022-09-19T01:11:16.655579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"π Combing All
","metadata":{}},{"cell_type":"markdown","source":"# 7. π Combing All","metadata":{}},{"cell_type":"code","source":"# Variant A\nclass AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):\n def __init__(self, models):\n self.models = models\n \n # we define clones of the original models to fit the data in\n def fit(self, X, y):\n self.models_ = [clone(x) for x in self.models]\n \n # Train cloned base models\n for model in self.models_:\n model_name = model.__class__.__name__\n model_details = str(model)\n print('model_name:', model_name)\n print('model_details:', model_details)\n #model.fit(X, y)\n if (\"KerasRegressor\" in model_name):\n model.fit(X, y, shuffle=True, validation_split=0.3, callbacks=callbacks_list) # fit the model for this fold\n if (\"keras\" in model_details):\n #model.fit(X, y, shuffle=True, validation_split=0.3, callbacks=callbacks_list) # fit the model for this fold\n model.fit(X, y, dnn__shuffle=True, dnn__validation_split=0.3, dnn__callbacks=callbacks_list)\n else:\n model.fit(X, y) # fit the model for this fold\n\n return self\n \n #Now we do the predictions for cloned models and average them\n def predict(self, X):\n predictions = np.column_stack([\n model.predict(X) for model in self.models_\n ])\n # IDEA: return weighted means\n return np.mean(predictions, axis=1)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:11:16.658247Z","iopub.execute_input":"2022-09-19T01:11:16.658671Z","iopub.status.idle":"2022-09-19T01:11:16.669153Z","shell.execute_reply.started":"2022-09-19T01:11:16.658639Z","shell.execute_reply":"2022-09-19T01:11:16.668207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Score","metadata":{}},{"cell_type":"code","source":"#use_voting_regressor = 0 # AveragingModels Averages\nuse_voting_regressor = 1 # VotingRegressor Averages \n#CBoost_new_10 = CatBoostRegressor(logging_level='Silent', random_state=10, depth=5, l2_leaf_reg=1.0, n_estimators=1700,eval_metric='RMSE',learning_rate=0.025,random_strength=3.7,bagging_temperature=1.0,grow_policy='SymmetricTree',bootstrap_type='Bayesian')\n\nwith MyTimer():\n if (use_voting_regressor == 1):\n print(\"running VotingRegressor\")\n from sklearn.ensemble import VotingRegressor\n estimator_list = [('CBoost', CBoost_new),('xgb',model_xgb),('ENet',ENet),('gboost',GBoost),('krr',KRR),('br',BR)]\n weight_list = [4,2,2,2,2,2]\n #estimator_list = [('dnn', dnn)]\n averaged_models = VotingRegressor(estimators=estimator_list, weights=weight_list) \n averaged_models.fit(train, y_train)\n averaged_train_pred = averaged_models.predict(train)\n averaged_pred = inv_boxcox1p(averaged_models.predict(test), lam_l)\n else:\n print(\"running AveragingModels\")\n #AveragingModels will fit and predict each model and predict using the mean of the individual predictions\n averaged_models = AveragingModels(models = (CBoost_new,CBoost_new,model_xgb,ENet,GBoost,KRR,BR))# ENet, model_xgb,dnn_pipe)) # Adding ENet and RF is worse, model_xgb_new is worse \n averaged_models.fit(train, y_train)\n averaged_train_pred = averaged_models.predict(train)\n averaged_pred = inv_boxcox1p(averaged_models.predict(test), lam_l)\n\nshow_metrics = 0\nif (show_metrics == 1):\n score = mae_cv(averaged_models)\n print(\" Averaged base models score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std()))\n score = rmsle_cv(averaged_models)\n print(\" Averaged base models score: {:.4f} ({:.4f})\\n\".format(score.mean(), score.std())) \n\n print(mae(y_train, averaged_train_pred))\n print(rmsle(y_train, averaged_train_pred))\nprint(averaged_pred)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:11:16.670918Z","iopub.execute_input":"2022-09-19T01:11:16.671456Z","iopub.status.idle":"2022-09-19T01:11:52.830879Z","shell.execute_reply.started":"2022-09-19T01:11:16.671276Z","shell.execute_reply":"2022-09-19T01:11:52.829634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ap=pd.DataFrame(averaged_pred)\nap.to_csv('ap2.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:11:52.832981Z","iopub.execute_input":"2022-09-19T01:11:52.833832Z","iopub.status.idle":"2022-09-19T01:11:52.851899Z","shell.execute_reply.started":"2022-09-19T01:11:52.833788Z","shell.execute_reply":"2022-09-19T01:11:52.850448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CBoost_new.fit(train,y_train)\nprint(CBoost_new.predict(test))\nmodel_xgb.fit(train,y_train)\nprint(model_xgb.predict(test))\nENet.fit(train,y_train)\nprint(ENet.predict(test))\nGBoost.fit(train,y_train)\nprint(GBoost.predict(test))\nKRR.fit(train,y_train)\nprint(KRR.predict(test))\nBR.fit(train,y_train)\nprint(BR.predict(test))","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:11:52.853637Z","iopub.execute_input":"2022-09-19T01:11:52.854298Z","iopub.status.idle":"2022-09-19T01:12:27.264429Z","shell.execute_reply.started":"2022-09-19T01:11:52.854260Z","shell.execute_reply":"2022-09-19T01:12:27.263064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"averaged_models.get_params()","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:12:27.266107Z","iopub.execute_input":"2022-09-19T01:12:27.266455Z","iopub.status.idle":"2022-09-19T01:12:27.327405Z","shell.execute_reply.started":"2022-09-19T01:12:27.266420Z","shell.execute_reply":"2022-09-19T01:12:27.326109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"π Compraing Predictions
","metadata":{}},{"cell_type":"markdown","source":"# π 8. Compraing Predictions","metadata":{}},{"cell_type":"code","source":"if (use_voting_regressor == 1):\n model_xgb.fit(train, y_train)\n model_lgb.fit(train, y_train)\n GBoost.fit(train, y_train)\n CBoost.fit(train, y_train)\n KRR.fit(train, y_train)\n BR.fit(train, y_train)\n averaged_models_temp = VotingRegressor(estimators=estimator_list, weights=weight_list) \n averaged_models_temp.fit(train, y_train)\n\n xt = test[:20]\n\n pred1 = model_xgb.predict(xt)\n pred2 = model_lgb.predict(xt)\n pred3 = GBoost.predict(xt)\n pred4 = CBoost.predict(xt)\n pred5 = KRR.predict(xt)\n pred6 = BR.predict(xt)\n pred7 = averaged_models_temp.predict(xt)\n plt.figure(figsize=(12,12))\n plt.plot(pred1, 'gd', label='XGBRegressor')\n plt.plot(pred2, 'bd', label='LGBRegressor')\n plt.plot(pred3, 'yd', label='GradientBoostingRegressor')\n plt.plot(pred4, 'rd', label='CatBoostRegressor')\n plt.plot(pred5, 'gs', label='KernelRidge')\n plt.plot(pred6, 'bs', label='BayesianRidge')\n plt.plot(pred7, 'r*', ms=10, label='VotingRegressor')\n\n plt.tick_params(axis='x', which='both', bottom=False, top=False,\n labelbottom=False)\n plt.ylabel('predicted')\n plt.xlabel('training samples')\n plt.legend(loc=\"best\")\n plt.title('Regressor predictions and their average')\n\n plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:12:27.329181Z","iopub.execute_input":"2022-09-19T01:12:27.329573Z","iopub.status.idle":"2022-09-19T01:13:37.095251Z","shell.execute_reply.started":"2022-09-19T01:12:27.329536Z","shell.execute_reply":"2022-09-19T01:13:37.094329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"π Stacking
","metadata":{}},{"cell_type":"markdown","source":"# π 9. Stacking","metadata":{}},{"cell_type":"code","source":"# Variant B\nclass StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):\n def __init__(self, base_models, meta_model, n_folds=10, shuffle=True): # increasing the n_folds value should give a more accurate prediction, averaged over n_fold iterations\n self.base_models = base_models\n self.meta_model = meta_model\n self.n_folds = n_folds\n self.shuffle = shuffle\n\n # Fit the data on clones of the original models\n def fit(self, X, y):\n self.base_models_ = [list() for x in self.base_models]\n self.meta_model_ = clone(self.meta_model)\n kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156) # was 156\n print(\"shuffle=\" + str(self.shuffle))\n \n # Train cloned base models then create out-of-fold predictions\n # that are needed to train the cloned meta-model\n out_of_fold_predictions = np.zeros((X.shape[0], len(self.base_models)))\n for i, model in enumerate(self.base_models): # for each model passed in\n with MyTimer():\n model_name = model.__class__.__name__\n model_details = str(model)\n print('model_name:', model_name)\n print('model_details:', model_details)\n for train_index, holdout_index in kfold.split(X, y): # create train,holdout splits for the number of folds\n instance = clone(model)\n self.base_models_[i].append(instance)\n #if (\"KerasRegressor\" in model_name):\n if (\"KerasRegressor\" in model_name):\n hist = instance.fit(X[train_index], y[train_index], shuffle=self.shuffle, validation_split=0.3, callbacks=callbacks_list) # fit the model for this fold\n elif (\"keras\" in model_details):\n instance.fit(X[train_index], y[train_index], dnn__shuffle=True, dnn__validation_split=0.3, dnn__callbacks=callbacks_list)\n else:\n instance.fit(X[train_index], y[train_index]) # fit the model for this fold\n y_pred = instance.predict(X[holdout_index]) # predict values for this fold\n out_of_fold_predictions[holdout_index, i] = y_pred # add predictions for this model and fold (random rows)\n #print('out_of_fold_predictions', out_of_fold_predictions)\n # Now train the cloned meta-model using the out-of-fold predictions as new and only feature\n print(\"out_of_fold_predictions\", out_of_fold_predictions)\n \n meta_model_name = self.meta_model_.__class__.__name__\n print(\"meta_model_name:\", meta_model_name)\n if (\"KerasRegressor\" in meta_model_name):\n self.meta_model_.fit(out_of_fold_predictions, y, shuffle=self.shuffle, validation_split=0.3, callbacks=callbacks_list) # need to see out_of_fold_predictions feature set\n elif (\"keras\" in str(self.meta_model_)):\n self.meta_model_.fit(out_of_fold_predictions, y, dnn__shuffle=True, dnn__validation_split=0.3, dnn__callbacks=callbacks_list) # need to see out_of_fold_predictions feature set\n else:\n self.meta_model_.fit(out_of_fold_predictions, y) # need to see out_of_fold_predictions feature set\n\n #self.meta_model_.fit(out_of_fold_predictions, y) # need to see out_of_fold_predictions feature set\n return self \n \n # Calculate the predictions of all base models on the test data and use the averaged predictions as \n # meta-features for the final prediction which is calculated by the meta-model\n \n # add MinMax\n def predict(self, X):\n # column_stack() function is used to stack 1-D arrays as columns into a 2-D array.\n meta_features = np.column_stack([\n np.column_stack([model.predict(X) for model in base_models]).mean(axis=1)\n for base_models in self.base_models_ ])\n return self.meta_model_.predict(meta_features)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:13:37.099386Z","iopub.execute_input":"2022-09-19T01:13:37.099745Z","iopub.status.idle":"2022-09-19T01:13:37.130974Z","shell.execute_reply.started":"2022-09-19T01:13:37.099709Z","shell.execute_reply":"2022-09-19T01:13:37.129917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stacked_averaged_models = StackingAveragedModels(base_models = (ENet, KRR, GBoost, model_xgb, lr_poly),#, dnn),\n meta_model = R_new, shuffle=True) \n\nif (compare_models == 1):\n with MyTimer():\n if (competition == 'SR'):\n score = mae_cv(stacked_averaged_models)\n print(\"Stacking Averaged models score mean and std: {:.4f} ({:.4f})\".format(score.mean(), score.std()))\n else:\n score = rmsle_cv(stacked_averaged_models)\n print(\"Stacking Averaged models score mean and std: {:.4f} ({:.4f})\".format(score.mean(), score.std()))","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:13:37.132537Z","iopub.execute_input":"2022-09-19T01:13:37.133053Z","iopub.status.idle":"2022-09-19T01:13:37.149200Z","shell.execute_reply.started":"2022-09-19T01:13:37.133003Z","shell.execute_reply":"2022-09-19T01:13:37.148022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"use_Regressor = 0\nif (use_Regressor == 1):\n from sklearn.ensemble import StackingRegressor\n from sklearn.linear_model import RidgeCV, LassoCV\n from sklearn.linear_model import ElasticNetCV\n\n use_cv = 1\n k = {'alpha': 2.2, 'coef0': 0.5, 'degree': 5, 'gamma': 0.001, 'kernel': 'polynomial'}\n if (use_cv == 1):\n e = {'fit_intercept': True, 'l1_ratio': 0.85, 'max_iter': 100, 'normalize': False, 'selection': 'random', 'cv': 10} # 'alpha': 0.05,\n r = {'fit_intercept': True, 'normalize': False, 'cv': None, 'gcv_mode': 'auto'} # cv value has no effect\n estimators = [('enet', make_pipeline(StandardScaler(), ElasticNetCV(**e, random_state=3))),\n ('gboost', GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05, max_depth=4, max_features='sqrt', min_samples_leaf=15, min_samples_split=10, loss='huber', random_state=5)),\n ('krr', KernelRidge(**k)),\n ('br', BayesianRidge())]\n reg = StackingRegressor(\n estimators=estimators,\n final_estimator=RidgeCV(**r))\n else:\n e = {'alpha': 0.05, 'fit_intercept': True, 'l1_ratio': 0.85, 'max_iter': 100, 'normalize': False, 'selection': 'random', 'tol': 0.05}\n r = {'alpha': 8, 'copy_X': True, 'fit_intercept': True, 'max_iter': None, 'normalize': False, 'solver': 'auto', 'tol': 0.05, 'cv': None, 'gcv_mode': 'auto', 'random_state': 99} # \n estimators = [('enet', make_pipeline(StandardScaler(), ElasticNet(**e, random_state=3))),\n ('gboost', GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05, max_depth=4, max_features='sqrt', min_samples_leaf=15, min_samples_split=10, loss='huber', random_state=5)),\n ('krr', KernelRidge(**k)),\n ('br', BayesianRidge())]\n reg = StackingRegressor(\n estimators=estimators,\n final_estimator=Ridge(**r))\n\n reg.fit(train, y_train)\n stacked_pred = inv_boxcox1p(reg.predict(test), lam_l)\n #reg.transform(inv_boxcox1p(stacked_averaged_models_temp, lam_l))\n print(stacked_pred)\n print(reg)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:13:37.151219Z","iopub.execute_input":"2022-09-19T01:13:37.151888Z","iopub.status.idle":"2022-09-19T01:13:37.166856Z","shell.execute_reply.started":"2022-09-19T01:13:37.151846Z","shell.execute_reply":"2022-09-19T01:13:37.166134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (use_Regressor == 2):\n from mlxtend.regressor import StackingCVRegressor\n from sklearn.linear_model import RidgeCV, LassoCV\n from sklearn.linear_model import ElasticNetCV\n\n use_cv = 0\n k = {'alpha': 2.2, 'coef0': 0.5, 'degree': 5, 'gamma': 0.001, 'kernel': 'polynomial'}\n if (use_cv == 1):\n e = {'fit_intercept': True, 'l1_ratio': 0.85, 'max_iter': 100, 'normalize': False, 'selection': 'random', 'cv': 10} # 'alpha': 0.05,\n r = {'fit_intercept': True, 'normalize': False, 'cv': None, 'gcv_mode': 'auto'} # cv value has no effect\n enet = make_pipeline(StandardScaler(), ElasticNetCV(**e, random_state=3))\n gboost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05, max_depth=4, max_features='sqrt', min_samples_leaf=15, min_samples_split=10, loss='huber', random_state=5)\n krr = KernelRidge(**k)\n br = BayesianRidge()\n r = RidgeCV(**r)\n reg = StackingCVRegressor(\n regressors=(enet, gboost, krr, br),\n meta_regressor=r)\n else:\n e = {'alpha': 0.05, 'fit_intercept': True, 'l1_ratio': 0.85, 'max_iter': 100, 'normalize': False, 'selection': 'random'}\n r = {'alpha': 8, 'copy_X': True, 'fit_intercept': True, 'max_iter': None, 'normalize': False, 'solver': 'auto', 'tol': 0.001, 'random_state': 99} # \n enet = make_pipeline(StandardScaler(), ElasticNet(**e, random_state=3))\n gboost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05, max_depth=4, max_features='sqrt', min_samples_leaf=15, min_samples_split=10, loss='huber', random_state=5)\n krr = KernelRidge(**k)\n br = BayesianRidge()\n r = Ridge(**r)\n reg = StackingCVRegressor(\n regressors=(enet, gboost, krr, br),\n meta_regressor=r)\n\n reg.fit(train, y_train)\n stacked_pred = inv_boxcox1p(reg.predict(test), lam_l)\n print(stacked_pred)\n print(reg)\n \n print('5-fold cross validation scores:\\n')\n for clf, label in zip([enet, gboost, krr, br], ['enet', 'gboost', \n 'krr', 'br',\n 'StackingCVRegressor']):\n scores = cross_val_score(clf, train, y_train, cv=5, scoring='neg_mean_squared_error')\n print(\"Neg. MSE Score: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label))\n \n #scores = cross_val_score(clf, train, y_train, cv=5)\n #print(\"R^2 Score: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label))","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:13:37.168513Z","iopub.execute_input":"2022-09-19T01:13:37.169275Z","iopub.status.idle":"2022-09-19T01:13:37.184441Z","shell.execute_reply.started":"2022-09-19T01:13:37.169235Z","shell.execute_reply":"2022-09-19T01:13:37.183262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"π Ensembling
","metadata":{}},{"cell_type":"markdown","source":"# π 9. Ensembling","metadata":{}},{"cell_type":"markdown","source":"## Process Stacked Regressor","metadata":{}},{"cell_type":"code","source":"averaged_models.fit(train, y_train)\naveraged_train_pred = averaged_models.predict(train)\nif (use_voting_regressor == 0):\n averaged_pred = inv_boxcox1p(averaged_models.predict(test), lam_l)\n\nif (competition == 'SR'):\n print(mae(y_train, averaged_train_pred))\nelse:\n print(rmsle(y_train, averaged_train_pred))\n \nplt.figure(figsize=(15,15))\nplt.scatter(averaged_train_pred, y_train, alpha=.7,\n color='b') #alpha helps to show overlapping data\nplt.xlabel('Predicted Price')\nplt.ylabel('Actual Price')\nplt.title('Averaged Model')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:13:37.186640Z","iopub.execute_input":"2022-09-19T01:13:37.187239Z","iopub.status.idle":"2022-09-19T01:14:12.407072Z","shell.execute_reply.started":"2022-09-19T01:13:37.187204Z","shell.execute_reply":"2022-09-19T01:14:12.405975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (method == \"stacked\"):\n with MyTimer():\n stacked_averaged_models.fit(train.values, y_train)\n with MyTimer():\n stacked_train_pred = stacked_averaged_models.predict(train.values)\n if (use_Regressor == 0):\n stacked_pred = inv_boxcox1p(stacked_averaged_models.predict(test.values), lam_l)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:14:12.408802Z","iopub.execute_input":"2022-09-19T01:14:12.409216Z","iopub.status.idle":"2022-09-19T01:20:31.613183Z","shell.execute_reply.started":"2022-09-19T01:14:12.409177Z","shell.execute_reply":"2022-09-19T01:20:31.611716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit_pred(train, y_train, test, model):\n #model.fit(train, y_train)\n \n model_name = model.__class__.__name__\n print('model_name:', model_name)\n if (\"XGBRegressor\" in model_name):\n model.fit(train, y_train) # fit the model for this fold\n elif (\"KerasRegressor\" in model_name):\n model.fit(train, y_train, shuffle=False, validation_split=0.3, callbacks=callbacks_list) # fit the model for this fold\n elif (\"keras\" in str(model)):\n model.fit(train, y_train, dnn__shuffle=True, dnn__validation_split=0.3, dnn__callbacks=callbacks_list) # fit the model for this fold\n else:\n print(\"train\", train)\n print(\"y_train\", y_train)\n model.fit(train, y_train) # fit the model for this fold\n\n model_train_pred = model.predict(train)\n model_pred = inv_boxcox1p(model.predict(test), lam_l)\n return(model_train_pred, model_pred)\n \nmodels = [ CBoost, lasso, lasso_new, ENet, KRR, GBoost, model_xgb, BR , AB, KN, B] # model_lgb,\nmodel_names = ['CBoost', 'lasso', 'lasso_new', 'ENet', 'KRR', 'GBoost', 'model_xgb', 'BR', 'AB', 'KN', 'B']\ndf_train_pred = pd.DataFrame()\ndf_test_pred = pd.DataFrame()\nwith MyTimer():\n for i in range(0,len(models)):\n #print(\"models[i]\", models[i])\n #print(\"model_names[i]:\", model_names[i])\n mn = model_names[i]+\"_pred\"\n train_pred, test_pred = fit_pred(train, y_train, test, models[i])\n df_train_pred[mn] = train_pred\n df_test_pred[mn] = test_pred\n #print(mn, test_pred) \n ","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:20:31.619775Z","iopub.execute_input":"2022-09-19T01:20:31.620508Z","iopub.status.idle":"2022-09-19T01:21:08.135058Z","shell.execute_reply.started":"2022-09-19T01:20:31.620457Z","shell.execute_reply":"2022-09-19T01:21:08.133968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Process XGB","metadata":{}},{"cell_type":"code","source":"replace_xgb = 0 # new optimized model is worse, was overfit\nif (replace_xgb == 1):\n model_xgb_new.fit(train, y_train)\n xgb_train_pred = model_xgb_new.predict(train)\n xgb_pred = inv_boxcox1p(model_xgb_new.predict(test), lam_l)\nelse:\n model_xgb.fit(train, y_train)\n xgb_train_pred = model_xgb.predict(train)\n xgb_pred = inv_boxcox1p(model_xgb.predict(test), lam_l)\n\nif (competition == 'SR'):\n print(mae(y_train, xgb_train_pred))\nelse:\n print(rmsle(y_train, xgb_train_pred))","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:21:08.136640Z","iopub.execute_input":"2022-09-19T01:21:08.137009Z","iopub.status.idle":"2022-09-19T01:21:20.290282Z","shell.execute_reply.started":"2022-09-19T01:21:08.136972Z","shell.execute_reply":"2022-09-19T01:21:20.289440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Process LGBM","metadata":{}},{"cell_type":"code","source":"model_lgb.fit(train, y_train)\nlgb_train_pred = model_lgb.predict(train)\nlgb_pred = inv_boxcox1p(model_lgb.predict(test), lam_l)\n\nif (competition == 'SR'):\n print(mae(y_train, lgb_train_pred))\nelse:\n print(rmsle(y_train, lgb_train_pred))","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:21:20.291948Z","iopub.execute_input":"2022-09-19T01:21:20.292346Z","iopub.status.idle":"2022-09-19T01:21:20.792109Z","shell.execute_reply.started":"2022-09-19T01:21:20.292311Z","shell.execute_reply":"2022-09-19T01:21:20.791110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (tuning_lgb == 1):\n model_lgb_op.fit(train, y_train)\n lgb_train_pred_op = model_lgb_op.predict(train)\n lgb_pred_op = inv_boxcox1p(model_lgb_op.predict(test), lam_l)\n\n if (competition == 'SR'):\n print(mae(y_train, lgb_train_pred_op))\n else:\n print(rmsle(y_train, lgb_train_pred_op))","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:21:20.795521Z","iopub.execute_input":"2022-09-19T01:21:20.797650Z","iopub.status.idle":"2022-09-19T01:21:20.804281Z","shell.execute_reply.started":"2022-09-19T01:21:20.797617Z","shell.execute_reply":"2022-09-19T01:21:20.803635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# compare values with optimization\nprint(lgb_train_pred)\nif (tuning_lgb == 1):\n print(lgb_train_pred_op)\nprint(lgb_pred)\nif (tuning_lgb == 1):\n print(lgb_pred_op)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:21:20.805881Z","iopub.execute_input":"2022-09-19T01:21:20.806281Z","iopub.status.idle":"2022-09-19T01:21:20.817606Z","shell.execute_reply.started":"2022-09-19T01:21:20.806242Z","shell.execute_reply":"2022-09-19T01:21:20.816685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('CBoost_new.pkl', 'rb') as fid:\n CBoost_new = pickle.load(fid)\n \nCBoost_new.fit\ncb_pred = inv_boxcox1p(CBoost_new.predict(test), lam_l)\ncb_train_pred = inv_boxcox1p(CBoost_new.predict(train), lam_l)\nprint(cb_pred)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:21:20.819158Z","iopub.execute_input":"2022-09-19T01:21:20.819580Z","iopub.status.idle":"2022-09-19T01:21:20.871544Z","shell.execute_reply.started":"2022-09-19T01:21:20.819532Z","shell.execute_reply":"2022-09-19T01:21:20.870414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ENet.fit(train,y_train)\nKRR.fit(train,y_train)\nGBoost.fit(train,y_train)\nlr_poly.fit(train,y_train)\nmodel_xgb.fit(train,y_train)\nenet_pred = inv_boxcox1p(ENet.predict(test), lam_l)\nkrr_pred = inv_boxcox1p(KRR.predict(test), lam_l)\ngboost_pred = inv_boxcox1p(GBoost.predict(test), lam_l)\nlr_pred = inv_boxcox1p(lr_poly.predict(test), lam_l)\n\nbr_pred = inv_boxcox1p(BR.predict(test), lam_l)\nbr_train_pred = inv_boxcox1p(BR.predict(train), lam_l)\n# all models being used in final model\n#print(cb_pred)\n#print(xgb_pred)\n#print(lgb_pred)\nprint(br_pred)\nprint(enet_pred) \nprint(krr_pred)\nprint(gboost_pred)\nprint(lr_pred) # changes each time\n#print(dnn_pred)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:21:20.872932Z","iopub.execute_input":"2022-09-19T01:21:20.873293Z","iopub.status.idle":"2022-09-19T01:22:00.345013Z","shell.execute_reply.started":"2022-09-19T01:21:20.873257Z","shell.execute_reply":"2022-09-19T01:22:00.343497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (method == 'stacked'):\n testing_ratio = 0\n if (testing_ratio == 1):\n stkr = 1.00\n xgbr = 0.00\n lgbr = 0.00\n cbr = 0.00\n brr = 0.00\n else:\n stkr = 0.70\n xgbr = 0.10 # .10\n lgbr = 0.00\n cbr = 0.20 # .20\n brr = 0.00 # 0\n\n '''RMSE on the entire Train data when averaging'''\n\n print('RMSLE score on train data:')\n print(mae(y_train,stacked_train_pred*stkr+ xgb_train_pred*xgbr + lgb_train_pred*lgbr + cb_train_pred*cbr + br_train_pred*brr ))\n print(rmsle(y_train,stacked_train_pred*stkr + xgb_train_pred*xgbr + lgb_train_pred*lgbr + cb_train_pred*cbr + br_train_pred*brr ))","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:00.352298Z","iopub.execute_input":"2022-09-19T01:22:00.356183Z","iopub.status.idle":"2022-09-19T01:22:00.377043Z","shell.execute_reply.started":"2022-09-19T01:22:00.356122Z","shell.execute_reply":"2022-09-19T01:22:00.375580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(method)\nif (method == 'stacked'):\n ensemble = stacked_pred*stkr + xgb_pred*xgbr + lgb_pred*lgbr + cb_pred*cbr + br_pred*brr # if using averaged_pred, need to add averaged_pred here\nelse:\n ensemble = averaged_pred","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:00.379058Z","iopub.execute_input":"2022-09-19T01:22:00.379775Z","iopub.status.idle":"2022-09-19T01:22:00.386599Z","shell.execute_reply.started":"2022-09-19T01:22:00.379739Z","shell.execute_reply":"2022-09-19T01:22:00.385653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(xgb_pred)\nprint(lgb_pred)\nprint(cb_pred)\nprint(br_pred)\nprint(averaged_pred)\nprint(ensemble)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:00.388193Z","iopub.execute_input":"2022-09-19T01:22:00.388841Z","iopub.status.idle":"2022-09-19T01:22:00.405952Z","shell.execute_reply.started":"2022-09-19T01:22:00.388805Z","shell.execute_reply":"2022-09-19T01:22:00.404733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if (method == 'stacked'):\n print(y_train,stacked_train_pred * stkr + xgb_train_pred * xgbr + lgb_train_pred * lgbr) # if using averaged_pred, need to add averaged_pred here\n print(y_train,stacked_train_pred)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:00.411913Z","iopub.execute_input":"2022-09-19T01:22:00.413860Z","iopub.status.idle":"2022-09-19T01:22:00.432214Z","shell.execute_reply.started":"2022-09-19T01:22:00.413810Z","shell.execute_reply":"2022-09-19T01:22:00.430437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_train = pd.DataFrame()\nsub_train['Id'] = train_ID\nif (method == 'stacked'):\n sub_train['SalePrice'] = inv_boxcox1p(stacked_train_pred, lam_l)\nelse:\n sub_train['SalePrice'] = inv_boxcox1p(averaged_train_pred, lam_l)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:00.437481Z","iopub.execute_input":"2022-09-19T01:22:00.438929Z","iopub.status.idle":"2022-09-19T01:22:00.459447Z","shell.execute_reply.started":"2022-09-19T01:22:00.438866Z","shell.execute_reply":"2022-09-19T01:22:00.457971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Predicted = sub_train['SalePrice']\nActual = inv_boxcox1p(y_train, lam_l)\nplt.figure(figsize=(15,15))\nplt.scatter(sub_train['SalePrice'], Actual, alpha=.7,\n color='b') #alpha helps to show overlapping data\nplt.xlabel('Predicted Price')\nplt.ylabel('Actual Price')\nplt.title('Averaged Model')\n\nm, b = np.polyfit(Predicted, Actual, 1)\n#m = slope, b=intercept\nplt.plot(Predicted, m*Predicted+b,c='red')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:00.460994Z","iopub.execute_input":"2022-09-19T01:22:00.461606Z","iopub.status.idle":"2022-09-19T01:22:00.790442Z","shell.execute_reply.started":"2022-09-19T01:22:00.461549Z","shell.execute_reply":"2022-09-19T01:22:00.789489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pre-adjustment score\nprint(\"mae for boxcox(SalePrice)\",mae(y_train, sub_train['SalePrice']))\nprint(\"mse for boxcox(SalePrice)\",rmsle(y_train, sub_train['SalePrice']))\nprint(\"mae for SalePrice\",mae(Actual, Predicted))\nprint(\"mse for SalePrice\",rmsle(Actual, Predicted))","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:00.791828Z","iopub.execute_input":"2022-09-19T01:22:00.792510Z","iopub.status.idle":"2022-09-19T01:22:00.802408Z","shell.execute_reply.started":"2022-09-19T01:22:00.792474Z","shell.execute_reply":"2022-09-19T01:22:00.801242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Findling best cutoff","metadata":{}},{"cell_type":"code","source":"def AdjustHigh(sub_train, y_train):\n AdjustedScores = []\n for i in np.arange(.994, 1.000, 0.01):\n for j in np.arange(1.00, 1.10, .01):\n\n q1 = sub_train['SalePrice'].quantile(0.0025)\n q2 = sub_train['SalePrice'].quantile(0.0045)\n q3 = sub_train['SalePrice'].quantile(i)\n\n #Verify the cutoffs for the adjustment\n # adjust at low end\n #sub_train['SalePrice2'] = sub_train['SalePrice'].apply(lambda x: x if x > q1 else x*0.79)\n #sub_train['SalePrice2'] = sub_train['SalePrice'].apply(lambda x: x if x > q2 else x*0.89)\n\n # adjust at high end\n sub_train['SalePrice2'] = sub_train['SalePrice'].apply(lambda x: x if x < q3 else x*j)\n\n Predicted = sub_train['SalePrice2']\n Actual = inv_boxcox1p(y_train, lam_l)\n\n # Pre-adjustment score\n #print(\"mae for boxcox(SalePrice)\",mae(y_train, sub_train['SalePrice2']))\n #print(\"mse for boxcox(SalePrice)\",rmsle(y_train, sub_train['SalePrice2']))\n #print(\"mae for SalePrice\",mae(Actual, Predicted))\n #print(\"mse for SalePrice\",rmsle(Actual, Predicted))\n\n AdjustedScores.append([i, j, mae(y_train, boxcox1p(sub_train['SalePrice2'], lam_l)), rmsle(y_train, boxcox1p(sub_train['SalePrice2'], lam_l)), mae(Actual, Predicted), rmsle(Actual, Predicted)])\n print(q1,q2,q3)\n df_adj = pd.DataFrame(AdjustedScores, columns=[\"QUANT\",\"COEF\",\"MAE_BC\",\"RMSE_BC\",\"MAE_SP\",\"RMSE_SP\"])\n print('quantiles vs coefficients')\n df_adj.sort_values(by=['RMSE_BC'])\n print(df_adj)\n df2 = df_adj.sort_values(by=['RMSE_BC']).reset_index()\n coef_hi = df2.COEF[0]\n quant_hi = df2.QUANT[0]\n return (coef_hi, quant_hi)\n\nch, qh = AdjustHigh(sub_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:00.803945Z","iopub.execute_input":"2022-09-19T01:22:00.805316Z","iopub.status.idle":"2022-09-19T01:22:00.875740Z","shell.execute_reply.started":"2022-09-19T01:22:00.805272Z","shell.execute_reply":"2022-09-19T01:22:00.874621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('ch',ch)\nprint('qh',qh)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:00.877146Z","iopub.execute_input":"2022-09-19T01:22:00.877511Z","iopub.status.idle":"2022-09-19T01:22:00.882884Z","shell.execute_reply.started":"2022-09-19T01:22:00.877476Z","shell.execute_reply":"2022-09-19T01:22:00.881926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"q1 = sub_train['SalePrice'].quantile(0.0015)\nq2 = sub_train['SalePrice'].quantile(0.01)\nq3 = sub_train['SalePrice'].quantile(qh)\n\nprint(q1,q2,q3)\n\nsub_train['SalePrice2'] = sub_train['SalePrice'].apply(lambda x: x if x < q3 else x*ch)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:00.884286Z","iopub.execute_input":"2022-09-19T01:22:00.885172Z","iopub.status.idle":"2022-09-19T01:22:00.903402Z","shell.execute_reply.started":"2022-09-19T01:22:00.885127Z","shell.execute_reply":"2022-09-19T01:22:00.902210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,15))\nplt.scatter(sub_train['SalePrice'], sub_train['SalePrice2'], alpha=.7,\n color='b') #alpha helps to show overlapping data\nplt.xlabel('Predicted Price')\nplt.ylabel('Adjusted Predicted Price')\nplt.title('Averaged Model')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:00.905607Z","iopub.execute_input":"2022-09-19T01:22:00.906178Z","iopub.status.idle":"2022-09-19T01:22:01.211686Z","shell.execute_reply.started":"2022-09-19T01:22:00.906144Z","shell.execute_reply":"2022-09-19T01:22:01.210614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_train.query(\"SalePrice != SalePrice2\")","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:01.213383Z","iopub.execute_input":"2022-09-19T01:22:01.214042Z","iopub.status.idle":"2022-09-19T01:22:01.229239Z","shell.execute_reply.started":"2022-09-19T01:22:01.214004Z","shell.execute_reply":"2022-09-19T01:22:01.228312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Predicted = sub_train['SalePrice2']\nActual = inv_boxcox1p(y_train, lam_l)\nplt.figure(figsize=(15,15))\nplt.scatter(sub_train['SalePrice2'], Actual, alpha=.7,\n color='b') #alpha helps to show overlapping data\nplt.xlabel('Adj Predicted Price')\nplt.ylabel('Actual Price')\nplt.title('Averaged Model')\n\nm, b = np.polyfit(Predicted, Actual, 1)\n#m = slope, b=intercept\nplt.plot(Predicted, m*Predicted+b,c='red')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:01.231983Z","iopub.execute_input":"2022-09-19T01:22:01.232811Z","iopub.status.idle":"2022-09-19T01:22:01.550433Z","shell.execute_reply.started":"2022-09-19T01:22:01.232775Z","shell.execute_reply":"2022-09-19T01:22:01.549523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Post adjustment for high score\nprint(\"mae for boxcox(SalePrice2)\",mae(y_train, sub_train['SalePrice2']))\nprint(\"mse for boxcox(SalePrice2)\",rmsle(y_train, sub_train['SalePrice2']))\nprint(\"mae for SalePrice2\",mae(Actual, Predicted))\nprint(\"mse for SalePrice2\",rmsle(Actual, Predicted))","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:01.551879Z","iopub.execute_input":"2022-09-19T01:22:01.552223Z","iopub.status.idle":"2022-09-19T01:22:01.561098Z","shell.execute_reply.started":"2022-09-19T01:22:01.552196Z","shell.execute_reply":"2022-09-19T01:22:01.560058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Cutoff at low","metadata":{}},{"cell_type":"code","source":"def AdjustLow(sub_train, y_train):\n AdjustedScores = []\n for i in np.arange(.00, .02, 0.001):\n for j in np.arange(.90, 1.10, 0.01):\n\n q1 = sub_train['SalePrice'].quantile(i)\n q2 = sub_train['SalePrice'].quantile(0.1)\n q3 = sub_train['SalePrice'].quantile(.995)\n\n #Verify the cutoffs for the adjustment\n #print(q1,q2,q3)\n # adjust at low end\n sub_train['SalePrice2'] = sub_train['SalePrice'].apply(lambda x: x if x > q1 else x*j)\n #sub_train['SalePrice2'] = sub_train['SalePrice'].apply(lambda x: x if x > q2 else x*0.89)\n\n # adjust at high end\n #sub_train['SalePrice2'] = sub_train['SalePrice'].apply(lambda x: x if x < q3 else x*j)\n\n Predicted = sub_train['SalePrice2']\n Actual = inv_boxcox1p(y_train, lam_l)\n\n # Pre-adjustment score\n #print(\"mae for boxcox(SalePrice)\",mae(y_train, sub_train['SalePrice2']))\n #print(\"mse for boxcox(SalePrice)\",rmsle(y_train, sub_train['SalePrice2']))\n #print(\"mae for SalePrice\",mae(Actual, Predicted))\n #print(\"mse for SalePrice\",rmsle(Actual, Predicted))\n\n AdjustedScores.append([i, j, mae(y_train, boxcox1p(sub_train['SalePrice2'], lam_l)), rmsle(y_train, boxcox1p(sub_train['SalePrice2'], lam_l)), mae(Actual, Predicted), rmsle(Actual, Predicted)])\n \n df_adj = pd.DataFrame(AdjustedScores, columns=[\"QUANT\",\"COEF\",\"MAE_BC\",\"RMSE_BC\",\"MAE_SP\",\"RMSE_SP\"])\n print('quantiles vs coefficients')\n df_adj.sort_values(by=['RMSE_BC'])\n print(df_adj)\n df2 = df_adj.sort_values(by=['RMSE_BC']).reset_index()\n coef_lo = df2.COEF[0]\n quant_lo = df2.QUANT[0]\n return (coef_lo, quant_lo)\n\ncl, ql = AdjustLow(sub_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:01.562836Z","iopub.execute_input":"2022-09-19T01:22:01.563644Z","iopub.status.idle":"2022-09-19T01:22:03.764249Z","shell.execute_reply.started":"2022-09-19T01:22:01.563603Z","shell.execute_reply":"2022-09-19T01:22:03.763051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('cl',cl)\nprint('ql',ql)\n\nq1 = sub_train['SalePrice'].quantile(ql)\nq2 = sub_train['SalePrice'].quantile(0.1)\nq3 = sub_train['SalePrice'].quantile(qh)\n\nprint(q1,q2,q3)\n\n\nsub_train['SalePrice2'] = sub_train['SalePrice'].apply(lambda x: x*cl if x < q1 else ( x*ch if x > q3 else x))\nsub_train.query(\"SalePrice != SalePrice2\")","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:03.766224Z","iopub.execute_input":"2022-09-19T01:22:03.766970Z","iopub.status.idle":"2022-09-19T01:22:03.790501Z","shell.execute_reply.started":"2022-09-19T01:22:03.766927Z","shell.execute_reply":"2022-09-19T01:22:03.789504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_train['SalePrice2'] = sub_train['SalePrice'].apply(lambda x: x*cl if x < q1 else ( x*ch if x > q3 else x))\n\nsub_train.query(\"SalePrice != SalePrice2\")","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:03.792103Z","iopub.execute_input":"2022-09-19T01:22:03.792466Z","iopub.status.idle":"2022-09-19T01:22:03.809513Z","shell.execute_reply.started":"2022-09-19T01:22:03.792432Z","shell.execute_reply":"2022-09-19T01:22:03.808370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,15))\nplt.scatter(sub_train['SalePrice'], sub_train['SalePrice2'], alpha=.7,\n color='b') #alpha helps to show overlapping data\nplt.xlabel('Predicted Price')\nplt.ylabel('Adjusted Predicted Price')\nplt.title('Averaged Model')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:03.811400Z","iopub.execute_input":"2022-09-19T01:22:03.812371Z","iopub.status.idle":"2022-09-19T01:22:04.126619Z","shell.execute_reply.started":"2022-09-19T01:22:03.812336Z","shell.execute_reply":"2022-09-19T01:22:04.125524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Predicted = sub_train['SalePrice2']\nActual = inv_boxcox1p(y_train, lam_l)\nplt.figure(figsize=(15,15))\nplt.scatter(sub_train['SalePrice2'], Actual, alpha=.7,\n color='b') #alpha helps to show overlapping data\nplt.xlabel('Adj Predicted Price')\nplt.ylabel('Actual Price')\nplt.title('Averaged Model')\n\nm, b = np.polyfit(Predicted, Actual, 1)\n#m = slope, b=intercept\nplt.plot(Predicted, m*Predicted+b,c='red')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:04.128101Z","iopub.execute_input":"2022-09-19T01:22:04.128452Z","iopub.status.idle":"2022-09-19T01:22:04.452631Z","shell.execute_reply.started":"2022-09-19T01:22:04.128416Z","shell.execute_reply":"2022-09-19T01:22:04.451575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_train.query(\"SalePrice != SalePrice2\")","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:04.453990Z","iopub.execute_input":"2022-09-19T01:22:04.455056Z","iopub.status.idle":"2022-09-19T01:22:04.469547Z","shell.execute_reply.started":"2022-09-19T01:22:04.455011Z","shell.execute_reply":"2022-09-19T01:22:04.468482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Post adjustment for low and high score\nprint(\"mae for boxcox(SalePrice2)\",mae(y_train, sub_train['SalePrice2']))\nprint(\"mse for boxcox(SalePrice2)\",rmsle(y_train, sub_train['SalePrice2']))\nprint(\"mae for SalePrice2\",mae(Actual, Predicted))\nprint(\"mse for SalePrice2\",rmsle(Actual, Predicted))","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:04.471158Z","iopub.execute_input":"2022-09-19T01:22:04.472048Z","iopub.status.idle":"2022-09-19T01:22:04.482384Z","shell.execute_reply.started":"2022-09-19T01:22:04.472010Z","shell.execute_reply":"2022-09-19T01:22:04.481237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"π Submission
","metadata":{}},{"cell_type":"markdown","source":"# π 10. Submission ","metadata":{}},{"cell_type":"code","source":"sub = pd.DataFrame()\nsub['Id'] = test_ID\nsub['SalePrice'] = ensemble\n\nadjust_both = 0\nadjust_low = 0\nadjust_high = 1\n\nq1 = sub['SalePrice'].quantile(ql)\nq2 = sub['SalePrice'].quantile(0.1)\nq3 = sub['SalePrice'].quantile(qh)\n\nif (adjust_both == 1):\n sub['SalePrice'] = sub['SalePrice'].apply(lambda x: x*cl if (x < q1) else ( x*ch if (x > q3) else x))\nelif (adjust_low == 1):\n sub['SalePrice'] = sub['SalePrice'].apply(lambda x: x if (x > q1) else x*cl) \nelif (adjust_high == 1):\n sub['SalePrice'] = sub['SalePrice'].apply(lambda x: x if (x < q3) else x*ch)\nelse:\n print(\"no adjustments made\")\n \nsub.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T01:22:04.484591Z","iopub.execute_input":"2022-09-19T01:22:04.485079Z","iopub.status.idle":"2022-09-19T01:22:04.506173Z","shell.execute_reply.started":"2022-09-19T01:22:04.485039Z","shell.execute_reply":"2022-09-19T01:22:04.505036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Upvote, CommentFork, Share
","metadata":{}},{"cell_type":"markdown","source":"## ","metadata":{}}]}