{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":9994,"databundleVersionId":752467,"sourceType":"competition"}],"dockerImageVersionId":30589,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# ML Final Project: ASHRAE Great Energy Predictor\nThe goal of this project is to use building and weather data to predict energy usage for a given building. We are given 3 data sets to clean and merge before training 4 types of models.\n","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-08T00:57:39.294785Z","iopub.execute_input":"2023-12-08T00:57:39.295175Z","iopub.status.idle":"2023-12-08T00:57:39.715749Z","shell.execute_reply.started":"2023-12-08T00:57:39.295146Z","shell.execute_reply":"2023-12-08T00:57:39.715003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\nimport warnings\nimport datetime\nimport gc\n\nfrom sklearn.metrics import mean_squared_error\nimport lightgbm as lgb\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom tqdm import tqdm\n\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:57:39.717378Z","iopub.execute_input":"2023-12-08T00:57:39.717834Z","iopub.status.idle":"2023-12-08T00:57:41.930834Z","shell.execute_reply.started":"2023-12-08T00:57:39.717804Z","shell.execute_reply":"2023-12-08T00:57:41.929977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_GEPIII = '/kaggle/input/ashrae-energy-prediction'","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:57:41.932256Z","iopub.execute_input":"2023-12-08T00:57:41.932670Z","iopub.status.idle":"2023-12-08T00:57:41.937593Z","shell.execute_reply.started":"2023-12-08T00:57:41.932633Z","shell.execute_reply":"2023-12-08T00:57:41.936603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_power_meter = pd.read_csv(os.path.join(path_GEPIII,'train.csv'))\ndf_meta = pd.read_csv(os.path.join(path_GEPIII,'building_metadata.csv'))\ndf_weather = pd.read_csv(os.path.join(path_GEPIII, \"weather_train.csv\"))","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:57:41.940502Z","iopub.execute_input":"2023-12-08T00:57:41.940838Z","iopub.status.idle":"2023-12-08T00:57:59.179408Z","shell.execute_reply.started":"2023-12-08T00:57:41.940810Z","shell.execute_reply":"2023-12-08T00:57:59.178480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utility Functions","metadata":{}},{"cell_type":"code","source":"# https://www.kaggle.com/code/aitude/ashrae-missing-weather-data-handling/notebook\ndef missing_statistics(df):    \n    statitics = pd.DataFrame(df.isnull().sum()).reset_index()\n    statitics.columns=['COLUMN NAME',\"MISSING VALUES\"]\n    statitics['TOTAL ROWS'] = df.shape[0]\n    statitics['% MISSING'] = round((statitics['MISSING VALUES']/statitics['TOTAL ROWS'])*100,2)\n    return statitics","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:57:59.180604Z","iopub.execute_input":"2023-12-08T00:57:59.181171Z","iopub.status.idle":"2023-12-08T00:57:59.186876Z","shell.execute_reply.started":"2023-12-08T00:57:59.181140Z","shell.execute_reply":"2023-12-08T00:57:59.186077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_meta = df_meta.merge(df_power_meter[['building_id']].drop_duplicates(), on='building_id')\n# df_meta['merged_id'] = df_meta['building_id'].astype('str') + '_' + df_power_meter['meter'].astype('str')\n# df_meta.head(10)\n\nprint(len(df_power_meter), len(df_meta), len(df_weather))","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:57:59.188259Z","iopub.execute_input":"2023-12-08T00:57:59.189111Z","iopub.status.idle":"2023-12-08T00:57:59.206615Z","shell.execute_reply.started":"2023-12-08T00:57:59.189073Z","shell.execute_reply":"2023-12-08T00:57:59.205376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Weather Data Cleanup\nThis is one of the main pre-processing steps, where we take each of the weather features and ensure that they can be read correctly by the model. The goal here is to remove outliers, remove entries with missing values, and ensure that the dataframe is correctly sized.","metadata":{}},{"cell_type":"code","source":"df_weather.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:57:59.208195Z","iopub.execute_input":"2023-12-08T00:57:59.208505Z","iopub.status.idle":"2023-12-08T00:57:59.249831Z","shell.execute_reply.started":"2023-12-08T00:57:59.208479Z","shell.execute_reply":"2023-12-08T00:57:59.248741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_statistics(df_weather)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:57:59.251135Z","iopub.execute_input":"2023-12-08T00:57:59.251471Z","iopub.status.idle":"2023-12-08T00:57:59.285264Z","shell.execute_reply.started":"2023-12-08T00:57:59.251443Z","shell.execute_reply":"2023-12-08T00:57:59.284172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_weather[\"datetime\"] = pd.to_datetime(df_weather[\"timestamp\"])\ndf_weather[\"day\"] = df_weather[\"datetime\"].dt.day\ndf_weather[\"week\"] = df_weather[\"datetime\"].dt.isocalendar().week\ndf_weather[\"month\"] = df_weather[\"datetime\"].dt.month","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:57:59.286843Z","iopub.execute_input":"2023-12-08T00:57:59.287251Z","iopub.status.idle":"2023-12-08T00:57:59.402789Z","shell.execute_reply.started":"2023-12-08T00:57:59.287222Z","shell.execute_reply":"2023-12-08T00:57:59.401639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_weather = df_weather.set_index(['site_id','day','month'])","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:57:59.408892Z","iopub.execute_input":"2023-12-08T00:57:59.409260Z","iopub.status.idle":"2023-12-08T00:57:59.436808Z","shell.execute_reply.started":"2023-12-08T00:57:59.409231Z","shell.execute_reply":"2023-12-08T00:57:59.435833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Air Temperature","metadata":{}},{"cell_type":"code","source":"air_temperature_filler = pd.DataFrame(df_weather.groupby(['site_id','day','month'])['air_temperature'].mean(),columns=[\"air_temperature\"])\ndf_weather.update(air_temperature_filler,overwrite=False)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:57:59.438491Z","iopub.execute_input":"2023-12-08T00:57:59.438961Z","iopub.status.idle":"2023-12-08T00:57:59.481601Z","shell.execute_reply.started":"2023-12-08T00:57:59.438904Z","shell.execute_reply":"2023-12-08T00:57:59.480634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cloud Coverage","metadata":{}},{"cell_type":"code","source":"cloud_coverage_filler = df_weather.groupby(['site_id','day','month'])['cloud_coverage'].mean()\n# Step 2\ncloud_coverage_filler = pd.DataFrame(cloud_coverage_filler.fillna(method='ffill'),columns=[\"cloud_coverage\"])\n\ndf_weather.update(cloud_coverage_filler,overwrite=False)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:57:59.483369Z","iopub.execute_input":"2023-12-08T00:57:59.483799Z","iopub.status.idle":"2023-12-08T00:57:59.520840Z","shell.execute_reply.started":"2023-12-08T00:57:59.483751Z","shell.execute_reply":"2023-12-08T00:57:59.519931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dew Temperature","metadata":{}},{"cell_type":"code","source":"dew_temperature_filler = pd.DataFrame(df_weather.groupby(['site_id','day','month'])['dew_temperature'].mean(),columns=[\"dew_temperature\"])\ndf_weather.update(dew_temperature_filler,overwrite=False)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:57:59.522529Z","iopub.execute_input":"2023-12-08T00:57:59.522935Z","iopub.status.idle":"2023-12-08T00:57:59.555405Z","shell.execute_reply.started":"2023-12-08T00:57:59.522896Z","shell.execute_reply":"2023-12-08T00:57:59.554496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sea level pressure","metadata":{}},{"cell_type":"code","source":"# Step 1\nsea_level_filler = df_weather.groupby(['site_id','day','month'])['sea_level_pressure'].mean()\n# Step 2\nsea_level_filler = pd.DataFrame(sea_level_filler.fillna(method='ffill'),columns=['sea_level_pressure'])\n\ndf_weather.update(sea_level_filler,overwrite=False)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:57:59.557025Z","iopub.execute_input":"2023-12-08T00:57:59.557660Z","iopub.status.idle":"2023-12-08T00:57:59.592446Z","shell.execute_reply.started":"2023-12-08T00:57:59.557620Z","shell.execute_reply":"2023-12-08T00:57:59.591225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Wind direction","metadata":{}},{"cell_type":"code","source":"wind_direction_filler =  pd.DataFrame(df_weather.groupby(['site_id','day','month'])['wind_direction'].mean(),columns=['wind_direction'])\ndf_weather.update(wind_direction_filler,overwrite=False)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:57:59.594130Z","iopub.execute_input":"2023-12-08T00:57:59.594716Z","iopub.status.idle":"2023-12-08T00:57:59.627754Z","shell.execute_reply.started":"2023-12-08T00:57:59.594651Z","shell.execute_reply":"2023-12-08T00:57:59.626754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Wind Speed","metadata":{}},{"cell_type":"code","source":"wind_speed_filler =  pd.DataFrame(df_weather.groupby(['site_id','day','month'])['wind_speed'].mean(),columns=['wind_speed'])\ndf_weather.update(wind_speed_filler,overwrite=False)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:57:59.629023Z","iopub.execute_input":"2023-12-08T00:57:59.629534Z","iopub.status.idle":"2023-12-08T00:57:59.660817Z","shell.execute_reply.started":"2023-12-08T00:57:59.629505Z","shell.execute_reply":"2023-12-08T00:57:59.659954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Precipitation","metadata":{}},{"cell_type":"code","source":"# Step 1\nprecip_depth_filler = df_weather.groupby(['site_id','day','month'])['precip_depth_1_hr'].mean()\n# Step 2\nprecip_depth_filler = pd.DataFrame(precip_depth_filler.fillna(method='ffill'),columns=['precip_depth_1_hr'])\n\ndf_weather.update(precip_depth_filler,overwrite=False)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:57:59.662115Z","iopub.execute_input":"2023-12-08T00:57:59.662840Z","iopub.status.idle":"2023-12-08T00:57:59.694708Z","shell.execute_reply.started":"2023-12-08T00:57:59.662809Z","shell.execute_reply":"2023-12-08T00:57:59.693596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_statistics(df_weather)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:57:59.696003Z","iopub.execute_input":"2023-12-08T00:57:59.696311Z","iopub.status.idle":"2023-12-08T00:57:59.727751Z","shell.execute_reply.started":"2023-12-08T00:57:59.696285Z","shell.execute_reply":"2023-12-08T00:57:59.726890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_weather.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:57:59.729139Z","iopub.execute_input":"2023-12-08T00:57:59.729524Z","iopub.status.idle":"2023-12-08T00:57:59.751486Z","shell.execute_reply.started":"2023-12-08T00:57:59.729499Z","shell.execute_reply":"2023-12-08T00:57:59.750748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_weather = df_weather.reset_index()\n# df_weather = df_weather.drop(['datetime','day','week','month'],axis=1)\n\nprint(len(df_weather))\ndf_weather.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:57:59.752899Z","iopub.execute_input":"2023-12-08T00:57:59.753224Z","iopub.status.idle":"2023-12-08T00:57:59.784157Z","shell.execute_reply.started":"2023-12-08T00:57:59.753196Z","shell.execute_reply":"2023-12-08T00:57:59.783067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Merge the Data\nGiven the three data sets, we have to use pandas to merge the data accordingly - this is just an extra step that is specific to this model - having all of the data in 1 data frame is crucial.","metadata":{}},{"cell_type":"code","source":"df_train = df_power_meter.merge(df_meta, left_on='building_id',right_on='building_id',how='left')\ndf_train = df_train.merge(df_weather,how='left',left_on=['site_id','timestamp'],right_on=['site_id','timestamp'])\n\ndf_train.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:57:59.785714Z","iopub.execute_input":"2023-12-08T00:57:59.786096Z","iopub.status.idle":"2023-12-08T00:58:11.446009Z","shell.execute_reply.started":"2023-12-08T00:57:59.786067Z","shell.execute_reply":"2023-12-08T00:58:11.444965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_statistics(df_train)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:58:11.447534Z","iopub.execute_input":"2023-12-08T00:58:11.447861Z","iopub.status.idle":"2023-12-08T00:58:16.089978Z","shell.execute_reply.started":"2023-12-08T00:58:11.447834Z","shell.execute_reply":"2023-12-08T00:58:16.088941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop year built and floor count columns\ndf_train = df_train.drop(columns=[\"year_built\", \"floor_count\", \"timestamp\", \"datetime\"], errors=\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:58:16.091991Z","iopub.execute_input":"2023-12-08T00:58:16.092546Z","iopub.status.idle":"2023-12-08T00:58:17.330716Z","shell.execute_reply.started":"2023-12-08T00:58:16.092507Z","shell.execute_reply":"2023-12-08T00:58:17.329751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.get_dummies(df_train, columns=[\"primary_use\"])\ndf_train.head(200)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:58:17.332137Z","iopub.execute_input":"2023-12-08T00:58:17.332468Z","iopub.status.idle":"2023-12-08T00:58:26.045099Z","shell.execute_reply.started":"2023-12-08T00:58:17.332440Z","shell.execute_reply":"2023-12-08T00:58:26.043912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Raw Meter Reading Statistics:\\n\",df_train[\"meter_reading\"].describe(), \"\\n\\n\")\nprint(\"Logarithmic Meter Reading Statistics:\\n\", np.log1p(df_train[\"meter_reading\"]).describe())","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:58:26.046564Z","iopub.execute_input":"2023-12-08T00:58:26.047017Z","iopub.status.idle":"2023-12-08T00:58:28.331363Z","shell.execute_reply.started":"2023-12-08T00:58:26.046976Z","shell.execute_reply":"2023-12-08T00:58:28.330261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train.dropna()\nmissing_statistics(df_train)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:58:28.332858Z","iopub.execute_input":"2023-12-08T00:58:28.333934Z","iopub.status.idle":"2023-12-08T00:58:32.212313Z","shell.execute_reply.started":"2023-12-08T00:58:28.333892Z","shell.execute_reply":"2023-12-08T00:58:32.211095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.to_pickle('train_df.pkl')\n# test_df.to_pickle('test_df.pkl')\ndel df_train\ngc.collect() # release unreferenced memory\ntrain_df = pd.read_pickle('train_df.pkl')\n# test_df = pd.read_pickle('test_df.pkl')","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:58:32.214208Z","iopub.execute_input":"2023-12-08T00:58:32.215049Z","iopub.status.idle":"2023-12-08T00:58:37.808566Z","shell.execute_reply.started":"2023-12-08T00:58:32.215007Z","shell.execute_reply":"2023-12-08T00:58:37.807549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = np.log1p(train_df[\"meter_reading\"])\nX = train_df.drop(columns=[\"meter_reading\"])\n\n# y_train.describe()\n\nfrom sklearn.model_selection import train_test_split\n\nX_train, _1, y_train, _2 = train_test_split(X, y, train_size=0.2, random_state=10)\ndel X,y,_1,_2","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:58:37.815142Z","iopub.execute_input":"2023-12-08T00:58:37.815470Z","iopub.status.idle":"2023-12-08T00:59:00.192846Z","shell.execute_reply.started":"2023-12-08T00:58:37.815444Z","shell.execute_reply":"2023-12-08T00:59:00.190630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Standard scaler for  testing\n\nfrom sklearn.preprocessing import StandardScaler\n\n# y = np.log1p(df_train[\"meter_reading\"])\n# X = df_train.drop(columns=[\"meter_reading\"])\n\n# newdf = X.drop(columns=[\"id\"])\ncols = X_train.columns.tolist()\nnewdf_std = StandardScaler().fit_transform(X_train)\n# becomes an array, only change to dataframe if asked\n\nX_train = pd.DataFrame(newdf_std,columns = cols)\nX_train.head()\n\n# X_train_nn, X_test_nn, y_train_nn, y_test_nn = train_test_split(X_train, y_train, train_size=0.3, random_state=10)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:59:00.195831Z","iopub.execute_input":"2023-12-08T00:59:00.196275Z","iopub.status.idle":"2023-12-08T00:59:12.781442Z","shell.execute_reply.started":"2023-12-08T00:59:00.196236Z","shell.execute_reply":"2023-12-08T00:59:12.780392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, train_size=0.13, random_state=10)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:59:12.782532Z","iopub.execute_input":"2023-12-08T00:59:12.783160Z","iopub.status.idle":"2023-12-08T00:59:15.538611Z","shell.execute_reply.started":"2023-12-08T00:59:12.783131Z","shell.execute_reply":"2023-12-08T00:59:15.537769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.describe()","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:59:15.540247Z","iopub.execute_input":"2023-12-08T00:59:15.541414Z","iopub.status.idle":"2023-12-08T00:59:16.220078Z","shell.execute_reply.started":"2023-12-08T00:59:15.541371Z","shell.execute_reply":"2023-12-08T00:59:16.218805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training\nWe can train our 3 models: Random Forest, LG Boost, and K-Fold algorithms. They are regression algorithms so they will be predicting a float variable which is the energy used by the building.","metadata":{}},{"cell_type":"code","source":"# len(newdf_final.columns)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:59:16.221412Z","iopub.execute_input":"2023-12-08T00:59:16.221807Z","iopub.status.idle":"2023-12-08T00:59:16.227378Z","shell.execute_reply.started":"2023-12-08T00:59:16.221776Z","shell.execute_reply":"2023-12-08T00:59:16.226221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Neural Network\n-disregard\n","metadata":{}},{"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense\n\n# dense means each node in one layer is connected to every other node in the following layer\n\nmodel = Sequential()\n\n# hidden layer 1\nmodel.add(Dense(20, input_dim=30, activation=\"relu\"))\n\n# hidden layer 2\nmodel.add(Dense(10, activation=\"relu\"))\n\n# output layer\nmodel.add(Dense(1, activation=\"sigmoid\"))","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:59:16.228702Z","iopub.execute_input":"2023-12-08T00:59:16.229055Z","iopub.status.idle":"2023-12-08T00:59:28.804682Z","shell.execute_reply.started":"2023-12-08T00:59:16.229017Z","shell.execute_reply":"2023-12-08T00:59:28.803480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n\n# # 1 epoch = back and forth 1 time, batch size divides the the data into X batches of ~= length\n# model.fit(newdf_final, y_train, epochs=1, batch_size=1)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:59:28.806479Z","iopub.execute_input":"2023-12-08T00:59:28.806860Z","iopub.status.idle":"2023-12-08T00:59:28.814889Z","shell.execute_reply.started":"2023-12-08T00:59:28.806829Z","shell.execute_reply":"2023-12-08T00:59:28.813677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# _, accuracy = model.evaluate(X_test, y_test)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:59:28.816985Z","iopub.execute_input":"2023-12-08T00:59:28.817505Z","iopub.status.idle":"2023-12-08T00:59:28.847726Z","shell.execute_reply.started":"2023-12-08T00:59:28.817460Z","shell.execute_reply":"2023-12-08T00:59:28.846726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(\"Accuracy after standardization: {:2f}\".format(accuracy * 100))","metadata":{"execution":{"iopub.status.busy":"2023-12-08T00:59:28.849363Z","iopub.execute_input":"2023-12-08T00:59:28.849823Z","iopub.status.idle":"2023-12-08T00:59:28.859148Z","shell.execute_reply.started":"2023-12-08T00:59:28.849782Z","shell.execute_reply":"2023-12-08T00:59:28.858136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Random Forest Regressor","metadata":{}},{"cell_type":"code","source":"%%time\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.metrics import accuracy_score\n\nrf = RandomForestRegressor(random_state=10).fit(X_train, y_train)\n\n\n# acc_score = round(accuracy_score(y_test, y_pred),2)\n# acc_score","metadata":{"execution":{"iopub.status.busy":"2023-12-08T01:04:08.255758Z","iopub.execute_input":"2023-12-08T01:04:08.256734Z","iopub.status.idle":"2023-12-08T01:15:04.830810Z","shell.execute_reply.started":"2023-12-08T01:04:08.256671Z","shell.execute_reply":"2023-12-08T01:15:04.829363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_rf = rf.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T01:15:04.833306Z","iopub.execute_input":"2023-12-08T01:15:04.833849Z","iopub.status.idle":"2023-12-08T01:19:47.470407Z","shell.execute_reply.started":"2023-12-08T01:15:04.833807Z","shell.execute_reply":"2023-12-08T01:19:47.469228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\nrmse_rf = round(mean_squared_error(y_test, y_pred_rf),2)\nrmse_rf","metadata":{"execution":{"iopub.status.busy":"2023-12-08T01:19:47.471874Z","iopub.execute_input":"2023-12-08T01:19:47.472267Z","iopub.status.idle":"2023-12-08T01:19:47.502297Z","shell.execute_reply.started":"2023-12-08T01:19:47.472200Z","shell.execute_reply":"2023-12-08T01:19:47.501234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# important_features_dict = {}\n# for X_train.columns, val in enumerate(rf.feature_importances_):\n#     important_features_dict[idx] = val\n\n# important_features_list = sorted(important_features_dict,\n#                                  key=important_features_dict.get,\n#                                  reverse=True)\n\n# print(f'5 most important features: {important_features_list[:5]}')\n\nimport operator\nfeat_names = X_train.columns\nrank_dict = dict(zip(feat_names, rf.feature_importances_)) # from lecture 22\nsorted_ranking = sorted(rank_dict.items(), key=operator.itemgetter(1), reverse=True)\n# print(sorted_ranking)\n# topfeat2 = sorted_ranking[0][0]","metadata":{"execution":{"iopub.status.busy":"2023-12-08T01:28:15.179465Z","iopub.execute_input":"2023-12-08T01:28:15.179940Z","iopub.status.idle":"2023-12-08T01:28:16.120713Z","shell.execute_reply.started":"2023-12-08T01:28:15.179884Z","shell.execute_reply":"2023-12-08T01:28:16.119534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for feature, importance in sorted_ranking:\n    print(\"{}\\t-\\t{:.5f}\".format(feature, importance))","metadata":{"execution":{"iopub.status.busy":"2023-12-08T01:29:27.014538Z","iopub.execute_input":"2023-12-08T01:29:27.015010Z","iopub.status.idle":"2023-12-08T01:29:27.021899Z","shell.execute_reply.started":"2023-12-08T01:29:27.014977Z","shell.execute_reply":"2023-12-08T01:29:27.020789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del rf","metadata":{"execution":{"iopub.status.busy":"2023-12-08T01:04:05.400740Z","iopub.status.idle":"2023-12-08T01:04:05.401419Z","shell.execute_reply.started":"2023-12-08T01:04:05.401186Z","shell.execute_reply":"2023-12-08T01:04:05.401214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del y_pred_rf","metadata":{"execution":{"iopub.status.busy":"2023-12-08T01:04:05.402561Z","iopub.status.idle":"2023-12-08T01:04:05.403025Z","shell.execute_reply.started":"2023-12-08T01:04:05.402822Z","shell.execute_reply":"2023-12-08T01:04:05.402842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Linear Regression\n-disregard\n","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## LG Boost","metadata":{}},{"cell_type":"code","source":"params = {'boosting_type': 'gbdt', 'objective': 'regression',\n          'metric': {'rmse'}, 'subsample_freq': 1, 'learning_rate': 0.3,\n          'bagging_freq': 5, 'num_leaves': 330, 'feature_fraction': 0.9, \n          'lambda_l1': 1, 'lambda_l2': 1}","metadata":{"execution":{"iopub.status.busy":"2023-12-08T01:31:02.094070Z","iopub.execute_input":"2023-12-08T01:31:02.094508Z","iopub.status.idle":"2023-12-08T01:31:02.100996Z","shell.execute_reply.started":"2023-12-08T01:31:02.094477Z","shell.execute_reply":"2023-12-08T01:31:02.099543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nlgb_train = lgb.Dataset(X_train, y_train)\nlgb_eval = lgb.Dataset(X_test, y_test) \ngbm = lgb.train(params, lgb_train, num_boost_round=500,\n                valid_sets=(lgb_train, lgb_eval),\n                early_stopping_rounds=50, verbose_eval = 50)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T01:31:04.187969Z","iopub.execute_input":"2023-12-08T01:31:04.188398Z","iopub.status.idle":"2023-12-08T01:32:41.495058Z","shell.execute_reply.started":"2023-12-08T01:31:04.188369Z","shell.execute_reply":"2023-12-08T01:32:41.493837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = gbm.predict(X_test)\n\n# acc_score = round(accuracy_score(y_test, y_pred),2)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T01:32:41.497267Z","iopub.execute_input":"2023-12-08T01:32:41.497619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\n\nrmse_lgb = round(mean_squared_error(y_test, y_pred),2)\nrmse_lgb","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(y_pred)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T01:04:05.414204Z","iopub.status.idle":"2023-12-08T01:04:05.415042Z","shell.execute_reply.started":"2023-12-08T01:04:05.414747Z","shell.execute_reply":"2023-12-08T01:04:05.414780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del y_pred, lgb_train, lgb_eval, gbm","metadata":{"execution":{"iopub.status.busy":"2023-12-08T01:04:05.416776Z","iopub.status.idle":"2023-12-08T01:04:05.417176Z","shell.execute_reply.started":"2023-12-08T01:04:05.416987Z","shell.execute_reply":"2023-12-08T01:04:05.417006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## KFold ","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import KFold\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.linear_model import LinearRegression","metadata":{"execution":{"iopub.status.busy":"2023-12-08T01:04:05.419059Z","iopub.status.idle":"2023-12-08T01:04:05.419738Z","shell.execute_reply.started":"2023-12-08T01:04:05.419487Z","shell.execute_reply":"2023-12-08T01:04:05.419518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ncv = KFold(n_splits=10, random_state=10, shuffle=True)\n\nlin = LinearRegression()\nscores = cross_val_score(lin, X_train, y_train, scoring=\"neg_mean_absolute_error\", cv=cv, n_jobs=-1)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T01:04:05.420917Z","iopub.status.idle":"2023-12-08T01:04:05.421759Z","shell.execute_reply.started":"2023-12-08T01:04:05.421502Z","shell.execute_reply":"2023-12-08T01:04:05.421528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.mean(np.absolute(scores))","metadata":{"execution":{"iopub.status.busy":"2023-12-08T01:04:05.423190Z","iopub.status.idle":"2023-12-08T01:04:05.423589Z","shell.execute_reply.started":"2023-12-08T01:04:05.423395Z","shell.execute_reply":"2023-12-08T01:04:05.423413Z"},"trusted":true},"execution_count":null,"outputs":[]}]}