{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# ASHRAE ENERGY PREDICTION III:","metadata":{"id":"PRDnm0lhzEyp"}},{"cell_type":"markdown","source":"# From Previous Notebook ....","metadata":{}},{"cell_type":"markdown","source":"## - Importing Necessary Libraries","metadata":{}},{"cell_type":"code","source":"# Importing needed libraries to be used throughout the project\n\n%matplotlib inline\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\n\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.preprocessing import FunctionTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn import utils\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import RandomizedSearchCV, GridSearchCV\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.linear_model import SGDRegressor,Ridge,ElasticNet\nfrom sklearn.model_selection import train_test_split,KFold,GroupKFold\nimport lightgbm as lgb\nimport gc\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn. linear_model import Lasso\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.svm import SVR\n\nimport tensorflow as tf\nfrom tensorflow import keras\nimport tensorflow.keras.layers as layers\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense,Dropout, Activation\nfrom tensorflow.keras.optimizers import *\n\nfrom prettytable import PrettyTable\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"id":"5h4-2ayuABWJ","execution":{"iopub.status.busy":"2022-01-13T13:55:11.061756Z","iopub.execute_input":"2022-01-13T13:55:11.062103Z","iopub.status.idle":"2022-01-13T13:55:19.439689Z","shell.execute_reply.started":"2022-01-13T13:55:11.062016Z","shell.execute_reply":"2022-01-13T13:55:19.438842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## - Loading and Reducing Memory Usage of Data","metadata":{"id":"OPc_EXwyA9b6"}},{"cell_type":"code","source":"data_path = \"/kaggle/input/ashrae-energy-prediction/\"\n\ntrain_path = data_path + \"train.csv\"\n\nbuilding_path = data_path + \"building_metadata.csv\"\n\nweather_train_path = data_path + \"weather_train.csv\"","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:55:22.875373Z","iopub.execute_input":"2022-01-13T13:55:22.875673Z","iopub.status.idle":"2022-01-13T13:55:22.881893Z","shell.execute_reply.started":"2022-01-13T13:55:22.875644Z","shell.execute_reply":"2022-01-13T13:55:22.881228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv(train_path)\n\nbuilding_data = pd.read_csv(building_path)\n\nweather_train_data = pd.read_csv(weather_train_path)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:55:26.117114Z","iopub.execute_input":"2022-01-13T13:55:26.117888Z","iopub.status.idle":"2022-01-13T13:55:41.365929Z","shell.execute_reply.started":"2022-01-13T13:55:26.117840Z","shell.execute_reply":"2022-01-13T13:55:41.365152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Converting data into feather format since some dataframes are too large and take a long time to load\n# This method is inspired from the kaggle notebook titled: ASHRAE: feather format for fast loading\n# Which is found at: https://www.kaggle.com/corochann/ashrae-feather-format-for-fast-loading\n\ntrain_data.to_feather('train_data.feather')\n\nbuilding_data.to_feather('building_data.feather')\n\nweather_train_data.to_feather('weather_train_data.feather')","metadata":{"id":"W3dUkNtVeSfP","execution":{"iopub.status.busy":"2022-01-13T13:55:41.367369Z","iopub.execute_input":"2022-01-13T13:55:41.367633Z","iopub.status.idle":"2022-01-13T13:55:44.313286Z","shell.execute_reply.started":"2022-01-13T13:55:41.367603Z","shell.execute_reply":"2022-01-13T13:55:44.312436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_feather('train_data.feather')\n\nbuilding_data = pd.read_feather('building_data.feather')\n\nweather_train_data = pd.read_feather('weather_train_data.feather')","metadata":{"id":"K-xSJPieegWJ","execution":{"iopub.status.busy":"2022-01-13T13:55:44.314440Z","iopub.execute_input":"2022-01-13T13:55:44.314676Z","iopub.status.idle":"2022-01-13T13:55:45.563796Z","shell.execute_reply.started":"2022-01-13T13:55:44.314647Z","shell.execute_reply":"2022-01-13T13:55:45.562915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This function helps in optimizing the memory used by the dataframes by by modifying/altering thedatatype of each column.\n# This method is inspired from the kaggle notebook titled: load data (reduce memory usage)\n# Which is found at: https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\n\ndef reduce_mem_usage(df, df_name):\n\n    start_mem = df.memory_usage().sum() / 1024**2\n    \n    for col in df.columns:\n\n        col_type = df[col].dtype\n\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n\n    print('Memory usage of {} is reduced by {:.2f} %. Usage dropped from {:.2f} MB to {:.2f} MB.'.format(df_name, (100 * (start_mem - end_mem) / start_mem), start_mem, end_mem))\n    \n    return df\n","metadata":{"id":"v3yIjOevKKKY","execution":{"iopub.status.busy":"2022-01-13T13:55:45.565632Z","iopub.execute_input":"2022-01-13T13:55:45.565869Z","iopub.status.idle":"2022-01-13T13:55:45.582019Z","shell.execute_reply.started":"2022-01-13T13:55:45.565840Z","shell.execute_reply":"2022-01-13T13:55:45.581042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reducing Memory Usage of Data\n\ntrain_data = reduce_mem_usage(train_data, 'Train Data')\n\nbuilding_data = reduce_mem_usage(building_data, 'Building Data')\n\nweather_train_data = reduce_mem_usage(weather_train_data, 'Weather Train Data')","metadata":{"id":"XYZL4jkJeVyB","outputId":"01e04941-ed2c-47a7-d13f-80fe439f5148","execution":{"iopub.status.busy":"2022-01-13T13:55:45.583432Z","iopub.execute_input":"2022-01-13T13:55:45.583968Z","iopub.status.idle":"2022-01-13T13:55:48.326758Z","shell.execute_reply.started":"2022-01-13T13:55:45.583926Z","shell.execute_reply":"2022-01-13T13:55:48.325896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merging Data\n\ntrain = train_data.merge(building_data, on='building_id', how='left')\ntrain = train.merge(weather_train_data, on=['site_id', 'timestamp'], how='left')","metadata":{"id":"y67TusNcgV3a","execution":{"iopub.status.busy":"2022-01-13T13:55:48.328193Z","iopub.execute_input":"2022-01-13T13:55:48.328668Z","iopub.status.idle":"2022-01-13T13:55:54.982204Z","shell.execute_reply.started":"2022-01-13T13:55:48.328629Z","shell.execute_reply":"2022-01-13T13:55:54.981440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Breaking Timestamp into Hour, Day, Month, Year\n\n# This function firstly converts timestamp to date and then breaks down date into 6 new columns: hour, day, dayOfWeek, dayOfYear, month and year\n\ndef breakdown_timestamp(dataframe):\n    \n    dataframe['timestamp']= pd.to_datetime(dataframe['timestamp'])\n\n    dataframe['hour']= np.uint8(dataframe['timestamp'].dt.hour)\n    \n    dataframe['day']= np.uint16(dataframe['timestamp'].dt.day)\n    dataframe['dayofweek']= np.uint8(dataframe['timestamp'].dt.dayofweek)\n    dataframe['dayofyear']= np.uint16(dataframe['timestamp'].dt.dayofyear)\n\n    dataframe['month']= np.uint8(dataframe['timestamp'].dt.month)\n\n    dataframe['year']= np.uint16(dataframe['timestamp'].dt.year)\n    \n    return dataframe","metadata":{"id":"6A03K8Z5uydO","execution":{"iopub.status.busy":"2022-01-13T13:55:54.983344Z","iopub.execute_input":"2022-01-13T13:55:54.983595Z","iopub.status.idle":"2022-01-13T13:55:54.991342Z","shell.execute_reply.started":"2022-01-13T13:55:54.983567Z","shell.execute_reply":"2022-01-13T13:55:54.990469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = breakdown_timestamp(train)","metadata":{"id":"0H9LBPwzuylB","execution":{"iopub.status.busy":"2022-01-13T13:55:54.992781Z","iopub.execute_input":"2022-01-13T13:55:54.993823Z","iopub.status.idle":"2022-01-13T13:56:07.304679Z","shell.execute_reply.started":"2022-01-13T13:55:54.993774Z","shell.execute_reply":"2022-01-13T13:56:07.303873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## - Applying Log Transformation to 'Meter Reading and 'Square Feet'","metadata":{}},{"cell_type":"code","source":"train['meter_reading'] = np.log1p(train['meter_reading'])","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:56:07.305851Z","iopub.execute_input":"2022-01-13T13:56:07.306086Z","iopub.status.idle":"2022-01-13T13:56:07.715849Z","shell.execute_reply.started":"2022-01-13T13:56:07.306057Z","shell.execute_reply":"2022-01-13T13:56:07.714943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['square_feet'] = np.log1p(train['square_feet'])","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:56:07.718294Z","iopub.execute_input":"2022-01-13T13:56:07.718552Z","iopub.status.idle":"2022-01-13T13:56:08.164866Z","shell.execute_reply.started":"2022-01-13T13:56:07.718522Z","shell.execute_reply":"2022-01-13T13:56:08.164128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## - Data Preperation and Feature Engineering","metadata":{}},{"cell_type":"code","source":"# Removing Zero Meter Readings\n\nzero_meter_readings = list(train[train['meter_reading'] == 0].index)\ntrain.drop(zero_meter_readings, axis = 0, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:56:08.165960Z","iopub.execute_input":"2022-01-13T13:56:08.166183Z","iopub.status.idle":"2022-01-13T13:56:14.214145Z","shell.execute_reply.started":"2022-01-13T13:56:08.166157Z","shell.execute_reply":"2022-01-13T13:56:14.213268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dropping Columns with More than 50 % Missing Values\n\nthreshold = len(train) * 0.5\ntrain.dropna(axis=1, thresh = threshold, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:56:14.215501Z","iopub.execute_input":"2022-01-13T13:56:14.215902Z","iopub.status.idle":"2022-01-13T13:56:16.270540Z","shell.execute_reply.started":"2022-01-13T13:56:14.215865Z","shell.execute_reply":"2022-01-13T13:56:16.269803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filling Missing Values\n\ntrain['cloud_coverage'].fillna(train['cloud_coverage'].median(), inplace=True)\ntrain['sea_level_pressure'].fillna(train['sea_level_pressure'].median(), inplace=True)\ntrain['precip_depth_1_hr'].fillna(train['precip_depth_1_hr'].median(), inplace=True)\ntrain['wind_direction'].fillna(train['wind_direction'].median(), inplace=True)\ntrain['wind_speed'].fillna(train['wind_speed'].median(), inplace=True)\ntrain['dew_temperature'].fillna(train['dew_temperature'].median(), inplace=True)\ntrain['air_temperature'].fillna(train['air_temperature'].median(), inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:56:16.271510Z","iopub.execute_input":"2022-01-13T13:56:16.272187Z","iopub.status.idle":"2022-01-13T13:56:18.984517Z","shell.execute_reply.started":"2022-01-13T13:56:16.272151Z","shell.execute_reply":"2022-01-13T13:56:18.983593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# adding new feature from existing ones to get better results\n\ntrain['season'] = train['timestamp'].apply(lambda x: 'Spring' if x.month==3 or x.month==4 or x.month==5 else \n                                                  'Summer' if x.month==6 or x.month==7 or x.month==8 else \n                                                  'Autumn' if x.month==9 or x.month==10 or x.month==11 else \n                                                  'Winter')\n\ntrain['isDayTime'] = train['timestamp'].apply(lambda x: 1 if x.hour >=6 and x.hour <=18 else 0)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:56:18.985819Z","iopub.execute_input":"2022-01-13T13:56:18.986143Z","iopub.status.idle":"2022-01-13T13:56:20.368807Z","shell.execute_reply.started":"2022-01-13T13:56:18.986109Z","shell.execute_reply":"2022-01-13T13:56:20.367863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Encoding categorical data\n\ncategorical_features = ['primary_use', 'season']\n\nencoder = preprocessing.LabelEncoder()\n\nfor i in categorical_features:\n    \n    train[i] = encoder.fit_transform(train[i])\n    ","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:56:20.369932Z","iopub.execute_input":"2022-01-13T13:56:20.370227Z","iopub.status.idle":"2022-01-13T13:56:32.591450Z","shell.execute_reply.started":"2022-01-13T13:56:20.370192Z","shell.execute_reply":"2022-01-13T13:56:32.590285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## - Dropping Timestamp and Reducing Memory Usage Again","metadata":{}},{"cell_type":"code","source":"train = train.drop(['timestamp'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:56:32.592935Z","iopub.execute_input":"2022-01-13T13:56:32.593218Z","iopub.status.idle":"2022-01-13T13:56:34.053367Z","shell.execute_reply.started":"2022-01-13T13:56:32.593185Z","shell.execute_reply":"2022-01-13T13:56:34.052538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reduced_train_data = reduce_mem_usage(train, 'Train Data')","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:56:34.054653Z","iopub.execute_input":"2022-01-13T13:56:34.054973Z","iopub.status.idle":"2022-01-13T13:56:38.164906Z","shell.execute_reply.started":"2022-01-13T13:56:34.054934Z","shell.execute_reply":"2022-01-13T13:56:38.163720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## - Removing Least Important Features Generated by our Feature Selection Method","metadata":{}},{"cell_type":"code","source":"new_data = reduced_train_data[['building_id','square_feet','primary_use','meter','site_id','air_temperature','dayofyear','hour','isDayTime','dew_temperature','dayofweek', 'meter_reading']]","metadata":{"id":"zn8fH0M72yG6","execution":{"iopub.status.busy":"2022-01-13T13:56:38.166317Z","iopub.execute_input":"2022-01-13T13:56:38.166711Z","iopub.status.idle":"2022-01-13T13:56:39.143625Z","shell.execute_reply.started":"2022-01-13T13:56:38.166666Z","shell.execute_reply":"2022-01-13T13:56:39.142731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_data.drop(['site_id','dew_temperature'],axis=1, inplace = True)","metadata":{"id":"oLudnHxa3B9P","execution":{"iopub.status.busy":"2022-01-13T13:56:39.145187Z","iopub.execute_input":"2022-01-13T13:56:39.145451Z","iopub.status.idle":"2022-01-13T13:56:39.640337Z","shell.execute_reply.started":"2022-01-13T13:56:39.145418Z","shell.execute_reply":"2022-01-13T13:56:39.639441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 8. Models and Fine Tuning","metadata":{"id":"Ucg_g-j-3qTs"}},{"cell_type":"code","source":"X_train = new_data.drop(['meter_reading'],axis = 1)\n\nY_train = new_data['meter_reading'].values","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:56:53.904310Z","iopub.execute_input":"2022-01-13T13:56:53.904654Z","iopub.status.idle":"2022-01-13T13:56:54.316812Z","shell.execute_reply.started":"2022-01-13T13:56:53.904620Z","shell.execute_reply":"2022-01-13T13:56:54.315773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8.1 Basline Model","metadata":{"id":"CJRp4quk4Q4e"}},{"cell_type":"code","source":"def baselineModel(y_actual,y_pred):\n\n    rmsle_score = np.sqrt(np.mean((y_actual - y_pred) * (y_actual-y_pred)))\n    \n    print(\"The RMSLE Score of the Baseline Model is :\",rmsle_score)\n\nbaselineModel(Y_train, np.median(Y_train))","metadata":{"id":"boJ171wC3uc1","execution":{"iopub.status.busy":"2022-01-13T13:56:56.918694Z","iopub.execute_input":"2022-01-13T13:56:56.918958Z","iopub.status.idle":"2022-01-13T13:56:58.203366Z","shell.execute_reply.started":"2022-01-13T13:56:56.918930Z","shell.execute_reply":"2022-01-13T13:56:58.202459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The baseline score can be computed is median value of labels. The baseline score for 50% of the data is 1.772. So it can be expected that the baseline score could be more.","metadata":{}},{"cell_type":"markdown","source":"**Splitting the data for training and testing**","metadata":{"id":"RzdWj-IO4byQ"}},{"cell_type":"code","source":"train_x, test_x, train_y, test_y = train_test_split(X_train,Y_train, test_size=0.2, random_state=42)","metadata":{"id":"VhSe5vVW4gMZ","execution":{"iopub.status.busy":"2022-01-13T13:57:06.985615Z","iopub.execute_input":"2022-01-13T13:57:06.985885Z","iopub.status.idle":"2022-01-13T13:57:11.169452Z","shell.execute_reply.started":"2022-01-13T13:57:06.985858Z","shell.execute_reply":"2022-01-13T13:57:11.168795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Calculating RMSLE**","metadata":{"id":"ckUtLIW34g_4"}},{"cell_type":"markdown","source":"RMSLE is the suggested evaluation metric for the models performance. Since we already applied log1p transformation to the target value 'Meter Reading', therefore, Root Mean Squared Error (RMSE) can be used directly. And can be computed as follows:","metadata":{}},{"cell_type":"code","source":"def RMSLE(y_actual, y_pred):\n    \n    return np.sqrt(mean_squared_error(y_actual, y_pred))","metadata":{"id":"PGw-MP784kfv","execution":{"iopub.status.busy":"2022-01-13T13:57:17.394610Z","iopub.execute_input":"2022-01-13T13:57:17.395267Z","iopub.status.idle":"2022-01-13T13:57:17.401278Z","shell.execute_reply.started":"2022-01-13T13:57:17.395225Z","shell.execute_reply":"2022-01-13T13:57:17.400332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8.2 Linear Regression","metadata":{"id":"w9iDSXmf4lj-"}},{"cell_type":"code","source":"linear_Regression = LinearRegression()\nlinear_Regression.fit(train_x, train_y)","metadata":{"id":"vy92R3c84oZR","execution":{"iopub.status.busy":"2022-01-13T13:57:39.090624Z","iopub.execute_input":"2022-01-13T13:57:39.091054Z","iopub.status.idle":"2022-01-13T13:57:42.059952Z","shell.execute_reply.started":"2022-01-13T13:57:39.091016Z","shell.execute_reply":"2022-01-13T13:57:42.059188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Calculating Prediction Score**","metadata":{"id":"ToTLBTVe4r47"}},{"cell_type":"code","source":"print('Linear Regression Traininig RMSLE = ', RMSLE((train_y) , (linear_Regression.predict(train_x))))\nprint('Linear Regression Testing RMSLE = ',RMSLE((test_y) ,(linear_Regression.predict(test_x))))","metadata":{"id":"qIFM-6944t9L","execution":{"iopub.status.busy":"2022-01-13T13:57:53.802633Z","iopub.execute_input":"2022-01-13T13:57:53.803314Z","iopub.status.idle":"2022-01-13T13:57:54.676761Z","shell.execute_reply.started":"2022-01-13T13:57:53.803265Z","shell.execute_reply":"2022-01-13T13:57:54.673862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Linear Regresssion Cross Validation**","metadata":{}},{"cell_type":"code","source":"lin_scores = cross_val_score(linear_Regression, train_x, train_y,scoring=\"neg_mean_squared_error\", cv=10)\nlin_rmse_scores = np.sqrt(-lin_scores)\n\nprint(\"Linear Regression CV Scores:\", lin_rmse_scores)\nprint(\"==========================================================================================\")\nprint(\"Mean CV Score:\", lin_rmse_scores.mean())","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:58:32.174174Z","iopub.execute_input":"2022-01-13T13:58:32.174487Z","iopub.status.idle":"2022-01-13T13:59:06.621377Z","shell.execute_reply.started":"2022-01-13T13:58:32.174452Z","shell.execute_reply":"2022-01-13T13:59:06.620315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8.3 ElasticNet","metadata":{"id":"YhrWMWcr4u7v"}},{"cell_type":"markdown","source":"**Hyperparameter Tuning**","metadata":{}},{"cell_type":"code","source":"parameters = {'alpha':[0.001, 0.01, 0.1, 1, 10, 100, 1000],\n              'fit_intercept' : [False],\n              'l1_ratio':[0.5]}\n\nelasticnet = GridSearchCV(estimator = ElasticNet(),\n                        param_grid = parameters,\n                        cv = 3, \n                        scoring = 'neg_mean_squared_error',\n                        verbose = 1,\n                        return_train_score = True,\n                        n_jobs = -1)\n\nelasticnet.fit(train_x, train_y)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:59:17.484507Z","iopub.execute_input":"2022-01-13T13:59:17.484797Z","iopub.status.idle":"2022-01-13T14:01:30.850627Z","shell.execute_reply.started":"2022-01-13T13:59:17.484767Z","shell.execute_reply":"2022-01-13T14:01:30.849545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best ElasticNet Estimator is : \", elasticnet.best_estimator_)\nprint(\"Best ElasticNet Paramteres are : \", elasticnet.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:01:46.874451Z","iopub.execute_input":"2022-01-13T14:01:46.874811Z","iopub.status.idle":"2022-01-13T14:01:46.882122Z","shell.execute_reply.started":"2022-01-13T14:01:46.874775Z","shell.execute_reply":"2022-01-13T14:01:46.880927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Building Best ElasticNet Estimator**","metadata":{}},{"cell_type":"code","source":"elastic_model = ElasticNet(alpha = 0.001, fit_intercept = False, l1_ratio = 0.5)\nelastic_model.fit(train_x, train_y)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:01:52.417520Z","iopub.execute_input":"2022-01-13T14:01:52.418402Z","iopub.status.idle":"2022-01-13T14:02:17.121225Z","shell.execute_reply.started":"2022-01-13T14:01:52.418334Z","shell.execute_reply":"2022-01-13T14:02:17.120444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Calculating Prediction Score**","metadata":{"id":"W35Ih6e55qvd"}},{"cell_type":"code","source":"print('ElasticNet Training RMSLE = ',RMSLE((train_y) , (elastic_model.predict(train_x))))\nprint('ElasticNet Testing RMSLE = ',RMSLE((test_y) , (elastic_model.predict(test_x))))","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:02:30.388459Z","iopub.execute_input":"2022-01-13T14:02:30.390255Z","iopub.status.idle":"2022-01-13T14:02:31.492304Z","shell.execute_reply.started":"2022-01-13T14:02:30.390173Z","shell.execute_reply":"2022-01-13T14:02:31.489623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8.4 Ridge","metadata":{"id":"LDNxyIkg5tKs"}},{"cell_type":"markdown","source":"**Hyperparameters Tuning**","metadata":{"id":"f4kMJoaY5yvi"}},{"cell_type":"code","source":"parameters = {'alpha':[0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000],\n              'fit_intercept' : [True],\n              'solver' : ['lsqr']}\n\nridge = GridSearchCV(estimator = Ridge(),\n                        param_grid = parameters,\n                        cv = 3, \n                        scoring = 'neg_mean_squared_error',\n                        verbose = 1,\n                        return_train_score = True,\n                        n_jobs = -1)\n\nridge.fit(train_x, train_y)","metadata":{"id":"VQHijdGf5wlD","execution":{"iopub.status.busy":"2022-01-13T14:03:23.327078Z","iopub.execute_input":"2022-01-13T14:03:23.327433Z","iopub.status.idle":"2022-01-13T14:04:06.117143Z","shell.execute_reply.started":"2022-01-13T14:03:23.327395Z","shell.execute_reply":"2022-01-13T14:04:06.115864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best Ridge Estimator is :\",ridge.best_estimator_)\nprint(\"Best Ridge Paramteres are : \", ridge.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:04:13.605429Z","iopub.execute_input":"2022-01-13T14:04:13.605718Z","iopub.status.idle":"2022-01-13T14:04:13.612729Z","shell.execute_reply.started":"2022-01-13T14:04:13.605690Z","shell.execute_reply":"2022-01-13T14:04:13.611816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Best Ridge Estimator**","metadata":{"id":"qs1qyc9w5yNb"}},{"cell_type":"code","source":"Ridge = Ridge(alpha = 0.0001, fit_intercept = True, solver = \"lsqr\")\nRidge.fit(train_x, train_y)","metadata":{"id":"RJ_aPX8b55OD","execution":{"iopub.status.busy":"2022-01-13T14:04:27.937630Z","iopub.execute_input":"2022-01-13T14:04:27.938121Z","iopub.status.idle":"2022-01-13T14:04:31.325246Z","shell.execute_reply.started":"2022-01-13T14:04:27.938070Z","shell.execute_reply":"2022-01-13T14:04:31.324355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Calculating Prediction Score**","metadata":{"id":"UlbOdwQU58O0"}},{"cell_type":"code","source":"print('Ridge Training RMSLE = ',RMSLE((train_y) , (Ridge.predict(train_x))))\nprint('Ridge Testing RMSLE = ',RMSLE(test_y , Ridge.predict(test_x)))","metadata":{"id":"LGYl4qJn5-PD","execution":{"iopub.status.busy":"2022-01-13T14:04:34.779159Z","iopub.execute_input":"2022-01-13T14:04:34.779866Z","iopub.status.idle":"2022-01-13T14:04:35.666711Z","shell.execute_reply.started":"2022-01-13T14:04:34.779825Z","shell.execute_reply":"2022-01-13T14:04:35.665677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8.5 Lasso","metadata":{}},{"cell_type":"markdown","source":"**Hyperparameter Tuninig**","metadata":{}},{"cell_type":"code","source":"parameters = {'alpha':[0.001, 0.01, 0.1, 1, 10]}\n\nlasso = GridSearchCV(estimator = Lasso(),\n                        param_grid = parameters,\n                        cv = 3, \n                        scoring = 'neg_mean_squared_error',\n                        verbose = 1,\n                        return_train_score = True,\n                        n_jobs = -1)\n\nlasso.fit(train_x, train_y)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:05:34.570728Z","iopub.execute_input":"2022-01-13T14:05:34.571670Z","iopub.status.idle":"2022-01-13T14:05:59.521033Z","shell.execute_reply.started":"2022-01-13T14:05:34.571613Z","shell.execute_reply":"2022-01-13T14:05:59.520038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best Lasso Estimator \",lasso.best_estimator_)\nprint(\"Best Lasso Paramteres are : \", lasso.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:06:21.249483Z","iopub.execute_input":"2022-01-13T14:06:21.249758Z","iopub.status.idle":"2022-01-13T14:06:21.255275Z","shell.execute_reply.started":"2022-01-13T14:06:21.249729Z","shell.execute_reply":"2022-01-13T14:06:21.254517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Best Lasso Estimator**","metadata":{}},{"cell_type":"code","source":"Lasso = Lasso(alpha = 0.001)\nLasso.fit(train_x, train_y)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:06:26.172689Z","iopub.execute_input":"2022-01-13T14:06:26.173363Z","iopub.status.idle":"2022-01-13T14:06:29.255606Z","shell.execute_reply.started":"2022-01-13T14:06:26.173317Z","shell.execute_reply":"2022-01-13T14:06:29.254660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Calculating Prediction Score**","metadata":{}},{"cell_type":"code","source":"print('Lasso Traininig RMSLE = ',RMSLE((train_y) , (Lasso.predict(train_x))))\nprint('Lasso Testing RMSLE = ',RMSLE(test_y , Lasso.predict(test_x)))","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:06:32.586051Z","iopub.execute_input":"2022-01-13T14:06:32.586350Z","iopub.status.idle":"2022-01-13T14:06:33.469088Z","shell.execute_reply.started":"2022-01-13T14:06:32.586317Z","shell.execute_reply":"2022-01-13T14:06:33.468135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8.6 Decision Tree","metadata":{"id":"dZ6s4mw45_Gf"}},{"cell_type":"markdown","source":"**Hyperparameters Tuning**","metadata":{"id":"LDzqmZ-L6fq-"}},{"cell_type":"code","source":"parameters = {'max_depth': [3,5,7,9,11,15]}\n\n\ndecission_tree = GridSearchCV(estimator = DecisionTreeRegressor(),\n                        param_grid = parameters,\n                        cv = 3, \n                        scoring = 'neg_mean_squared_error',\n                        verbose = 1,\n                        return_train_score = True,\n                        n_jobs = -1)\n\ndecission_tree.fit(train_x, train_y)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:07:12.558242Z","iopub.execute_input":"2022-01-13T14:07:12.558590Z","iopub.status.idle":"2022-01-13T14:15:07.276457Z","shell.execute_reply.started":"2022-01-13T14:07:12.558557Z","shell.execute_reply":"2022-01-13T14:15:07.275450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best Decision Tree Estimator \",decission_tree.best_estimator_)\nprint(\"Best Decision Tree Paramteres are : \", decission_tree.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:15:45.240796Z","iopub.execute_input":"2022-01-13T14:15:45.241184Z","iopub.status.idle":"2022-01-13T14:15:45.250108Z","shell.execute_reply.started":"2022-01-13T14:15:45.241143Z","shell.execute_reply":"2022-01-13T14:15:45.249136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Best Decision Tree Estimator**","metadata":{}},{"cell_type":"code","source":"DecissionTree = DecisionTreeRegressor(max_depth=15)\nDecissionTree.fit(train_x, train_y)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:15:52.343005Z","iopub.execute_input":"2022-01-13T14:15:52.343429Z","iopub.status.idle":"2022-01-13T14:17:47.741585Z","shell.execute_reply.started":"2022-01-13T14:15:52.343399Z","shell.execute_reply":"2022-01-13T14:17:47.740556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Calculating Prediction Score**","metadata":{}},{"cell_type":"code","source":"print('Decision Tree Training RMSLE = ',RMSLE((train_y) , (DecissionTree.predict(train_x))))\nprint('Decision Tree Testing RMSLE = ',RMSLE(test_y , DecissionTree.predict(test_x)))","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:18:05.391438Z","iopub.execute_input":"2022-01-13T14:18:05.391732Z","iopub.status.idle":"2022-01-13T14:18:09.969838Z","shell.execute_reply.started":"2022-01-13T14:18:05.391701Z","shell.execute_reply":"2022-01-13T14:18:09.968872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8.7 RandomForestRegressor","metadata":{}},{"cell_type":"markdown","source":"**NOTE: Running this model will take so much time and may lead to crashing the session in process.**","metadata":{}},{"cell_type":"markdown","source":"**Hyperparameters Tuning**","metadata":{}},{"cell_type":"code","source":"parameters = { 'n_estimators': [60,80,100],\n              'max_depth':[5,7,9]}\n\nforest_reg = GridSearchCV(estimator = RandomForestRegressor(),\n                        param_grid = parameters,\n                        cv = 3, \n                        scoring = 'neg_mean_squared_error',\n                        verbose = 1,\n                        return_train_score = True,\n                        n_jobs = -1)\nforest_reg.fit(train_x, train_y)","metadata":{"id":"mge8qJ716fSr","execution":{"iopub.status.busy":"2022-01-13T11:46:30.769410Z","iopub.status.idle":"2022-01-13T11:46:30.769817Z","shell.execute_reply.started":"2022-01-13T11:46:30.769590Z","shell.execute_reply":"2022-01-13T11:46:30.769611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best Random Forest Estimator \",forest_reg.best_estimator_)\nprint(\"Best Random Forest Paramteres are : \", forest_reg.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T11:46:30.770633Z","iopub.status.idle":"2022-01-13T11:46:30.770963Z","shell.execute_reply.started":"2022-01-13T11:46:30.770807Z","shell.execute_reply":"2022-01-13T11:46:30.770829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Best RandomForest Estimator**","metadata":{"id":"CK0Z1seB6lkr"}},{"cell_type":"code","source":"# RandomForest = RandomForestRegressor(n_estimators = n_estimators, max_depth = max_depth)\n# RandomForest.fit(train_x, train_y)","metadata":{"id":"7AXTdaUx6nE7","execution":{"iopub.status.busy":"2022-01-13T14:23:38.636440Z","iopub.execute_input":"2022-01-13T14:23:38.637588Z","iopub.status.idle":"2022-01-13T14:23:38.641202Z","shell.execute_reply.started":"2022-01-13T14:23:38.637540Z","shell.execute_reply":"2022-01-13T14:23:38.640371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Calculating Prediction Score**","metadata":{"id":"XrMwQn8l6tC-"}},{"cell_type":"code","source":"# print('Train RMSLE = ',RMSLE((train_y) , (RandomForest.predict(train_x))))\n# print('Test RMSLE = ',RMSLE(test_y , RandomForest.predict(test_x)))","metadata":{"id":"deDGfbYy6uUs","execution":{"iopub.status.busy":"2022-01-13T14:23:42.878559Z","iopub.execute_input":"2022-01-13T14:23:42.878854Z","iopub.status.idle":"2022-01-13T14:23:42.883180Z","shell.execute_reply.started":"2022-01-13T14:23:42.878824Z","shell.execute_reply":"2022-01-13T14:23:42.882274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8.8 SGD Regressor","metadata":{}},{"cell_type":"code","source":"# Scaling the Data Prior to Passing it to the Model\n\nx_train_scaled = preprocessing.scale(train_x)\nx_test_scaled = preprocessing.scale(test_x)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:25:49.901061Z","iopub.execute_input":"2022-01-13T14:25:49.901435Z","iopub.status.idle":"2022-01-13T14:25:54.747314Z","shell.execute_reply.started":"2022-01-13T14:25:49.901397Z","shell.execute_reply":"2022-01-13T14:25:54.746445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Hyperparameters Tuning**","metadata":{}},{"cell_type":"code","source":"parameters = {'alpha':[0.0001, 0.001],\n             'eta0': [0.001],\n             'penalty': ['l2'],\n             'learning_rate': ['adaptive', 'invscaling'],\n             'early_stopping': [True]}\n\nsgd = GridSearchCV(estimator = SGDRegressor(),\n                        param_grid = parameters,\n                        cv = 3, \n                        scoring = 'neg_mean_squared_error',\n                        verbose = 10,\n                        return_train_score = True,\n                        n_jobs = -1)\n\nsgd.fit(x_train_scaled, train_y)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:26:01.416263Z","iopub.execute_input":"2022-01-13T14:26:01.416552Z","iopub.status.idle":"2022-01-13T14:32:25.902255Z","shell.execute_reply.started":"2022-01-13T14:26:01.416524Z","shell.execute_reply":"2022-01-13T14:32:25.901168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best SGDRegressor Estimator \",sgd.best_estimator_)\nprint(\"Best SGDRegressor Paramteres are : \", sgd.best_params_)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:32:32.192806Z","iopub.execute_input":"2022-01-13T14:32:32.193088Z","iopub.status.idle":"2022-01-13T14:32:32.199899Z","shell.execute_reply.started":"2022-01-13T14:32:32.193059Z","shell.execute_reply":"2022-01-13T14:32:32.199177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Best SGDRegressor**","metadata":{}},{"cell_type":"code","source":"sgd_regressor = SGDRegressor(early_stopping=True, eta0=0.001, learning_rate='adaptive')\nsgd_regressor.fit(x_train_scaled, train_y)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:40:45.078631Z","iopub.execute_input":"2022-01-13T14:40:45.079553Z","iopub.status.idle":"2022-01-13T14:42:57.087573Z","shell.execute_reply.started":"2022-01-13T14:40:45.079511Z","shell.execute_reply":"2022-01-13T14:42:57.086235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Calculating Prediction Score**","metadata":{}},{"cell_type":"code","source":"print('SGDRegressor Training RMSLE = ',RMSLE((train_y) , (sgd_regressor.predict(x_train_scaled))))\nprint('SGDRegressor Testing RMSLE = ',RMSLE(test_y , sgd_regressor.predict(x_test_scaled)))","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:43:20.770626Z","iopub.execute_input":"2022-01-13T14:43:20.770917Z","iopub.status.idle":"2022-01-13T14:43:22.298943Z","shell.execute_reply.started":"2022-01-13T14:43:20.770888Z","shell.execute_reply":"2022-01-13T14:43:22.295831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8.9 SVR","metadata":{}},{"cell_type":"markdown","source":"**NOTE: Running this model will take so much time and may lead to crashing the session in process.**","metadata":{}},{"cell_type":"markdown","source":"**Building SVR Model**","metadata":{}},{"cell_type":"code","source":"svm_reg = SVR(kernel=\"linear\")\nsvm_reg.fit(x_train_scaled, train_y)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T11:46:30.779972Z","iopub.status.idle":"2022-01-13T11:46:30.780258Z","shell.execute_reply.started":"2022-01-13T11:46:30.780113Z","shell.execute_reply":"2022-01-13T11:46:30.780129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Calculating Prediction Score**","metadata":{}},{"cell_type":"code","source":"print('SVR Traininig RMSLE = ', RMSLE((train_y) , (svm_reg.predict(x_train_scaled))))\nprint('SVR Testing RMSLE = ',RMSLE((test_y) ,(svm_reg.predict(x_test_scaled))))","metadata":{"execution":{"iopub.status.busy":"2022-01-13T11:46:30.780950Z","iopub.status.idle":"2022-01-13T11:46:30.781220Z","shell.execute_reply.started":"2022-01-13T11:46:30.781082Z","shell.execute_reply":"2022-01-13T11:46:30.781097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8.10 3 Layer Neural Network","metadata":{}},{"cell_type":"code","source":"from keras import backend as K\n\ndef NN_RMSLE(y_actual, y_pred):\n    \n    return K.sqrt(K.mean(K.square(y_pred - y_actual))) ","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:53:28.363613Z","iopub.execute_input":"2022-01-13T14:53:28.364330Z","iopub.status.idle":"2022-01-13T14:53:28.369022Z","shell.execute_reply.started":"2022-01-13T14:53:28.364287Z","shell.execute_reply":"2022-01-13T14:53:28.368441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Splitting Training Data into Trainig and Validation Sets\n\ntrain_xx, val_xx, train_yy, val_yy = train_test_split(train_x,train_y, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:53:31.349615Z","iopub.execute_input":"2022-01-13T14:53:31.349943Z","iopub.status.idle":"2022-01-13T14:53:35.018650Z","shell.execute_reply.started":"2022-01-13T14:53:31.349908Z","shell.execute_reply":"2022-01-13T14:53:35.017698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.callbacks import EarlyStopping\n\nmodel = Sequential()\n\nearlyStop= EarlyStopping(monitor='val_loss', mode='min', patience=3)\n\nmodel.add(layers.Dense(512, activation='relu',input_shape=(train_xx.shape[1],)))\nmodel.add(Dropout(0.5))\nmodel.add(layers.Dense(256, activation='relu'))\nmodel.add(Dropout(0.2))\nmodel.add(layers.Dense(64, activation='relu'))\nmodel.add(Dropout(0.2))\nmodel.add(Dense(1  ,activation='linear'))\n\nmodel.compile(optimizer='adam', loss = NN_RMSLE)\n\nmodel.fit(train_xx, train_yy, epochs = 15, batch_size = 2048, validation_data=(val_xx,val_yy),callbacks = earlyStop)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T14:55:18.701501Z","iopub.execute_input":"2022-01-13T14:55:18.701800Z","iopub.status.idle":"2022-01-13T15:13:01.219333Z","shell.execute_reply.started":"2022-01-13T14:55:18.701772Z","shell.execute_reply":"2022-01-13T15:13:01.218186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Neural Network Training RMSLE = ', model.evaluate(train_x, train_y, verbose=0))\nprint('Neural Network Testing RMSLE = ', model.evaluate(test_x, test_y, verbose=0))","metadata":{"execution":{"iopub.status.busy":"2022-01-13T15:15:28.447153Z","iopub.execute_input":"2022-01-13T15:15:28.447805Z","iopub.status.idle":"2022-01-13T15:28:48.287142Z","shell.execute_reply.started":"2022-01-13T15:15:28.447765Z","shell.execute_reply":"2022-01-13T15:28:48.286476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8.11 Scores Comparison","metadata":{}},{"cell_type":"code","source":"scores_table = PrettyTable([\"Model Name\",\"Training RMSLE\",\"Testing RMSLE\"])\n\nscores_table.add_row([\"LinearRegression\", \"1.3811028\", \"1.3812215\"])\nscores_table.add_row([\"ElasticNet\", \"1.4735792\", \"1.4732091\"])\nscores_table.add_row([\"Ridge\", \"1.3842031\", \"1.3842150\"])\nscores_table.add_row([\"Lasso\", \"1.3811055\", \"1.3812206\"])\nscores_table.add_row([\"DecisionTree\", \"0.6077021\", \"0.6098925\"])\nscores_table.add_row([\"RandomForestRegressor\", \"--------\", \"--------\"])\nscores_table.add_row([\"SGDRegressor\", \"1.3811040\", \"1.3812219\"])\nscores_table.add_row([\"SVR\", \"--------\", \"--------\"])\nscores_table.add_row([\"Neural Network\", \"2.2980730\", \"2.2978658\"])\n\nprint(scores_table)","metadata":{"id":"P9wj_tW0DrtP","execution":{"iopub.status.busy":"2022-01-13T15:31:46.370845Z","iopub.execute_input":"2022-01-13T15:31:46.371678Z","iopub.status.idle":"2022-01-13T15:31:46.380544Z","shell.execute_reply.started":"2022-01-13T15:31:46.371637Z","shell.execute_reply":"2022-01-13T15:31:46.379485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**observations:**\n1. Out of all tested models, Decision Tree Regressor performed the best with an approximate RMSLE score of 0.6.\n2. Linear Regression, Ridge, Lasso and SGDRegressor had compitetively close scores, with Linear Regression talking the lead as the second best prediction method.\n3. Our constructed Neural Network Performed worst out of all tested models, indicating its unsuitability for such problem.","metadata":{}},{"cell_type":"markdown","source":"# 9.  Analyzing the Best Model","metadata":{}},{"cell_type":"markdown","source":"##  9.1 Viewing the Scores of the Hyperparameter Combinations Tested During the Grid Search","metadata":{}},{"cell_type":"code","source":"pd.DataFrame(decission_tree.cv_results_)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T16:14:49.818480Z","iopub.execute_input":"2022-01-13T16:14:49.818943Z","iopub.status.idle":"2022-01-13T16:14:49.844524Z","shell.execute_reply.started":"2022-01-13T16:14:49.818910Z","shell.execute_reply":"2022-01-13T16:14:49.843700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9.2 Viewing the Relative Importance of Each Attribute for Making Accurate Predictions","metadata":{}},{"cell_type":"code","source":"features_list = ['building_id','square_feet','primary_use','meter','air_temperature','dayofyear','hour','isDayTime','dayofweek']\n\nfeatures_importance = decission_tree.best_estimator_.feature_importances_\n\nsorted(zip(features_importance, features_list), reverse=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T16:16:31.819548Z","iopub.execute_input":"2022-01-13T16:16:31.819880Z","iopub.status.idle":"2022-01-13T16:16:31.829796Z","shell.execute_reply.started":"2022-01-13T16:16:31.819845Z","shell.execute_reply":"2022-01-13T16:16:31.828880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Observations**\n1. Square Feet attribute contributes the most towards predicting the amount of energy consumption.\n2. Building Id is the second highly influencing factor, with an importance of 0.25.\n3. It seems that attributes relating to time and date do not affect energy consumption much.","metadata":{}},{"cell_type":"markdown","source":"**Ensemble Methods**\n- We believe that Random Forest Regressor might offer better performance as they rely on Decision Trees and this ensemble can further enhance performance. Unfortunately, due to extremely slow running times/sessions crashing, we were unable to test this theory.","metadata":{}}]}