{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\n# Pre-processing libraries\n\"\"\"Import basic modules.\"\"\"\nimport pandas as pd              # For data manipulation\nimport numpy as np               # For linear algebra\nimport matplotlib.pyplot as plt  # For 2D visualization\nimport seaborn as sns            \nfrom scipy import stats          # For statistics\n%matplotlib inline\n\nfrom scipy.stats import kurtosis, skew\n\n\n\"\"\"Plotly visualization.\"\"\"\nimport plotly.graph_objs as go\nfrom plotly.tools import make_subplots\nfrom plotly.offline import iplot, init_notebook_mode\ninit_notebook_mode(connected = True)\n\n\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import power_transform\nfrom sklearn.preprocessing import OneHotEncoder\nfrom scipy import stats\nfrom datetime import datetime\n\n\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n        \n# setting pd to show all columns when using df.head()\npd.set_option(\"display.max_columns\", None)\n# '/kaggle/input/ashrae-energy-prediction/weather_train_filled.csv'","metadata":{"execution":{"iopub.status.busy":"2022-01-05T14:14:10.494358Z","iopub.execute_input":"2022-01-05T14:14:10.494689Z","iopub.status.idle":"2022-01-05T14:14:10.521895Z","shell.execute_reply.started":"2022-01-05T14:14:10.494653Z","shell.execute_reply":"2022-01-05T14:14:10.521099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Helping functions \n* Decrease memory usage\n*  calc the percnatge of null and nan in dataframe\n* chechk for imlbalnce in each of the dependet variabeles \n* chechk for outliers using boxplot \n","metadata":{"execution":{"iopub.status.busy":"2022-01-02T14:28:59.844042Z","iopub.execute_input":"2022-01-02T14:28:59.844366Z","iopub.status.idle":"2022-01-02T14:28:59.851315Z","shell.execute_reply.started":"2022-01-02T14:28:59.844333Z","shell.execute_reply":"2022-01-02T14:28:59.850045Z"}}},{"cell_type":"markdown","source":"## Minimizing memory usage as possible","metadata":{}},{"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:38:01.60165Z","iopub.execute_input":"2022-01-05T13:38:01.601958Z","iopub.status.idle":"2022-01-05T13:38:01.617442Z","shell.execute_reply.started":"2022-01-05T13:38:01.601928Z","shell.execute_reply":"2022-01-05T13:38:01.616386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# description : calc the missing data (nan && null) in the dataset and show it in the form of table \n# function 1 \ndef draw_missing_data_table(df):\n    total = df.isnull().sum().sort_values(ascending=False) # sum the number of null \n    percent = (df.isnull().sum()/df.isnull().count()).sort_values(ascending=False) # calc the percenatge \n    missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\n    return missing_data","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:38:01.762053Z","iopub.execute_input":"2022-01-05T13:38:01.762389Z","iopub.status.idle":"2022-01-05T13:38:01.768663Z","shell.execute_reply.started":"2022-01-05T13:38:01.762352Z","shell.execute_reply":"2022-01-05T13:38:01.767693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# function 3:\ndef removeOutliers(variable):\n    # Calculate 1st, 3rd quartiles and inter-quartile range.\n    q1, q3 = variable.quantile(0.25), variable.quantile(0.75)\n    iqr = q3 - q1\n    \n    # cacluating both fences of the boxplot (lower and upper )\n    upperFence, lowerFence = q3+1.5*iqr ,q1-1.5*iqr\n    \n    # based on the IQR we can find the outliers in the variables \n    outliers = variable[(variable > upperFence) | (variable < lowerFence)]    \n    \n    # drop outliers based on any data larger or smllaer than (upper or lower)fence\n    filterd = variable.drop(outliers.index, axis = 0)\n    \n    return filterd","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:38:01.910694Z","iopub.execute_input":"2022-01-05T13:38:01.910947Z","iopub.status.idle":"2022-01-05T13:38:01.917205Z","shell.execute_reply.started":"2022-01-05T13:38:01.910919Z","shell.execute_reply":"2022-01-05T13:38:01.916228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def outlier_plotbox(variable,filteredVariable):\n      # intgtare the function with boxplot to viulize\n    \n    # Create subplot object.\n \n    fig  = make_subplots(\n        rows= 2,\n        cols=1,\n        print_grid= False,\n        subplot_titles = (\"Distribution with Outliers\", \" Distribution without Outliers\"))\n    # This trace plots boxplot without outliers\n    fig.add_trace(\n        go.Box(\n            x = filteredVariable,\n            name = \"\",\n            marker = dict(color=\"green\")\n        ),\n    row=2,col=1)\n        \n    # This trace plots boxplot with outliers\n    fig.add_trace(\n        go.Box(\n            x = variable, # on \n            name = \"\", # This removes trace 0\n            marker = dict(color=\"darkred\")\n        ),\n    row=1,col=1)\n    \n    # Update layout\n    fig.layout.update(\n        height=800, \n        width=870,\n        showlegend=False,\n        paper_bgcolor=\"rgb(243, 243, 243)\",\n        plot_bgcolor=\"rgb(243, 243, 243)\"\n        )\n    \n    # Update axes\n    fig.layout.xaxis2.update(title=f\"<b>{variable.name}</b>\")\n    return fig.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:38:02.056637Z","iopub.execute_input":"2022-01-05T13:38:02.056929Z","iopub.status.idle":"2022-01-05T13:38:02.067134Z","shell.execute_reply.started":"2022-01-05T13:38:02.056894Z","shell.execute_reply":"2022-01-05T13:38:02.065774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading/merging data","metadata":{}},{"cell_type":"code","source":"\nbuilding_metadata = pd.read_csv('/kaggle/input/ashrae-energy-prediction/building_metadata.csv', \n                                dtype={'building_id':np.uint16, 'site_id':np.uint8})\nweather_train = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_train.csv', \n                            dtype={'site_id':np.uint8, 'air_temperature':np.float16,\n                            'cloud_coverage':np.float16, 'dew_temperature':np.float16,\n                            'precip_depth_1_hr':np.float16})\ntrain = pd.read_csv('/kaggle/input/ashrae-energy-prediction/train.csv', dtype={'building_id':np.uint16, 'meter':np.uint8})\n\n# Merging\ntrain = train.merge(building_metadata, on='building_id', how='left')\ntrain = train.merge(weather_train, on=['site_id', 'timestamp'], how='left')\n\n# Changes: I didnt drop any columns as I will need site_id to change faulty readings mentioned in the competetion\n# regarding site 0\n","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-01-05T13:38:02.401419Z","iopub.execute_input":"2022-01-05T13:38:02.402426Z","iopub.status.idle":"2022-01-05T13:38:24.266689Z","shell.execute_reply.started":"2022-01-05T13:38:02.402373Z","shell.execute_reply":"2022-01-05T13:38:24.265607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reduce_mem_usage(train)","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:38:24.26812Z","iopub.execute_input":"2022-01-05T13:38:24.268556Z","iopub.status.idle":"2022-01-05T13:38:27.18129Z","shell.execute_reply.started":"2022-01-05T13:38:24.268526Z","shell.execute_reply":"2022-01-05T13:38:27.180672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:38:27.182311Z","iopub.execute_input":"2022-01-05T13:38:27.182635Z","iopub.status.idle":"2022-01-05T13:38:27.187763Z","shell.execute_reply.started":"2022-01-05T13:38:27.182605Z","shell.execute_reply":"2022-01-05T13:38:27.186826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:38:27.190334Z","iopub.execute_input":"2022-01-05T13:38:27.1907Z","iopub.status.idle":"2022-01-05T13:38:27.223152Z","shell.execute_reply.started":"2022-01-05T13:38:27.190669Z","shell.execute_reply":"2022-01-05T13:38:27.222235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Agenda \n* splitting data \n* Imputation\n* Skwennes \n* Outliers \n* feature eng + extraction \n","metadata":{}},{"cell_type":"markdown","source":"### Evluating the amount of missing data in the data frame ","metadata":{}},{"cell_type":"code","source":"draw_missing_data_table(train)","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:38:27.22463Z","iopub.execute_input":"2022-01-05T13:38:27.22485Z","iopub.status.idle":"2022-01-05T13:38:43.020148Z","shell.execute_reply.started":"2022-01-05T13:38:27.224823Z","shell.execute_reply":"2022-01-05T13:38:43.019247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# target isolation \ny = train.iloc[:,3]\ntrain.drop(labels= ['meter_reading'],inplace = True, axis = 1)\n\n# Dropping ids as we will not need them anymore\ntrain.drop(columns=[\"building_id\"], inplace=True)\ntrain.drop(columns=[\"site_id\"], inplace=True) # is needed","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:38:43.021528Z","iopub.execute_input":"2022-01-05T13:38:43.022071Z","iopub.status.idle":"2022-01-05T13:38:48.079652Z","shell.execute_reply.started":"2022-01-05T13:38:43.022026Z","shell.execute_reply":"2022-01-05T13:38:48.078745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Findings : There are 2 features which havve a high missing values percntage, Threshold = 50%","metadata":{"execution":{"iopub.status.busy":"2022-01-01T22:23:13.54263Z","iopub.execute_input":"2022-01-01T22:23:13.542992Z","iopub.status.idle":"2022-01-01T22:23:28.465571Z","shell.execute_reply.started":"2022-01-01T22:23:13.542951Z","shell.execute_reply":"2022-01-01T22:23:28.464742Z"}}},{"cell_type":"markdown","source":"### findings : Drop the 2 features with more than 50% missing ( floor_count, year_built)","metadata":{}},{"cell_type":"code","source":"train.drop(labels = ['floor_count','year_built'],inplace = True, axis = 1)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-01-05T13:38:48.081048Z","iopub.execute_input":"2022-01-05T13:38:48.081415Z","iopub.status.idle":"2022-01-05T13:38:49.174485Z","shell.execute_reply.started":"2022-01-05T13:38:48.08136Z","shell.execute_reply":"2022-01-05T13:38:49.173481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# splitting data train,test,validate\n# 100%\n# 70-30 \n# 30*15 , 30*85\nx_train,x_test, y_train, y_test = train_test_split(train,y, test_size = 0.3,random_state = 4,shuffle = True)\nx_test, x_val, y_test, y_val = train_test_split(x_test, y_test, test_size=0.50, random_state=4 ,shuffle = True ) ","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:38:49.177041Z","iopub.execute_input":"2022-01-05T13:38:49.177335Z","iopub.status.idle":"2022-01-05T13:39:04.116056Z","shell.execute_reply.started":"2022-01-05T13:38:49.177305Z","shell.execute_reply":"2022-01-05T13:39:04.115258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install datawig","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.655887Z","iopub.status.idle":"2022-01-05T13:37:57.65633Z","shell.execute_reply.started":"2022-01-05T13:37:57.656125Z","shell.execute_reply":"2022-01-05T13:37:57.656151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n### * Imputation","metadata":{}},{"cell_type":"code","source":"import datawig\ntrain_knn = train.copy(deep=True)\ndf_train, df_test = datawig.utils.random_split(train)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.657679Z","iopub.status.idle":"2022-01-05T13:37:57.658001Z","shell.execute_reply.started":"2022-01-05T13:37:57.65783Z","shell.execute_reply":"2022-01-05T13:37:57.657852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### imputing the cloud_coverage as a test ","metadata":{}},{"cell_type":"code","source":"\nimputer = datawig.SimpleImputer(\n    input_columns = ['building_id', 'meter', 'timestamp', 'site_id', 'primary_use',\n       'square_feet', 'cloud_coverage' ,'dew_temperature',\n       'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction',\n       'wind_speed'],\n    output_column = 'air_temperature',\n    output_path = 'model'\n)","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.659554Z","iopub.status.idle":"2022-01-05T13:37:57.660031Z","shell.execute_reply.started":"2022-01-05T13:37:57.659822Z","shell.execute_reply":"2022-01-05T13:37:57.659853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imputer.fit(train_df=df_train, num_epochs=50)\n#Impute missing values and return original dataframe with predictions\nimputed = imputer.predict(df_test)","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.660908Z","iopub.status.idle":"2022-01-05T13:37:57.661647Z","shell.execute_reply.started":"2022-01-05T13:37:57.661443Z","shell.execute_reply":"2022-01-05T13:37:57.661466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### We did not import the test data, as the memory got full after applying features transformtion, especially OHE. As the feature 'primary_use' added 16 columns in both train and test which took huge amount of memory (Check previous versions), and we did not want to apply label encoder because in this way we will have ordinal data, which is not our case.","metadata":{}},{"cell_type":"markdown","source":"### Evluating the train data ","metadata":{}},{"cell_type":"markdown","source":"### skewness and kurtosis","metadata":{}},{"cell_type":"code","source":"display(\"Skwness is :\", y_train.skew(),\n       \"Kurtosis is :\", y_train.kurt())\nsns.distplot(y_train, hist = True ,color='red', label=['denisty','price'])\n# highly skweed data \n(y_test == 0).sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:39:04.11752Z","iopub.execute_input":"2022-01-05T13:39:04.117754Z","iopub.status.idle":"2022-01-05T13:39:51.671466Z","shell.execute_reply.started":"2022-01-05T13:39:04.117727Z","shell.execute_reply":"2022-01-05T13:39:51.670508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"square root tranfomration","metadata":{}},{"cell_type":"code","source":"y_train_sqrt = np.sqrt(y_train)\ndisplay(\"Skwness is :\", y_train_sqrt.skew(),\n       \"Kurtosis is :\", y_train_sqrt.kurt())\nsns.distplot(y_train_sqrt, hist = True ,color='blue', label=['denisty'])","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:39:51.673592Z","iopub.execute_input":"2022-01-05T13:39:51.673843Z","iopub.status.idle":"2022-01-05T13:40:39.49908Z","shell.execute_reply.started":"2022-01-05T13:39:51.673814Z","shell.execute_reply":"2022-01-05T13:40:39.496754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_log = y_train.copy(deep = True)\ny_train_log_new = y_train_log+1\ny_train_log = np.log(y_train_log_new)","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:41:59.350403Z","iopub.execute_input":"2022-01-05T13:41:59.350694Z","iopub.status.idle":"2022-01-05T13:41:59.44381Z","shell.execute_reply.started":"2022-01-05T13:41:59.350665Z","shell.execute_reply":"2022-01-05T13:41:59.443029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(\"Skwness is :\", y_train_log.skew(),\n       \"Kurtosis is :\", y_train_log.kurt())\nsns.distplot(y_train_log ,color='black' ,label=['denisty'], hist=False, kde=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-05T14:06:01.585791Z","iopub.execute_input":"2022-01-05T14:06:01.586267Z","iopub.status.idle":"2022-01-05T14:06:55.52268Z","shell.execute_reply.started":"2022-01-05T14:06:01.586222Z","shell.execute_reply":"2022-01-05T14:06:55.521275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Box-cox tranfomration ","metadata":{"execution":{"iopub.status.busy":"2022-01-05T12:33:19.608807Z","iopub.execute_input":"2022-01-05T12:33:19.609093Z","iopub.status.idle":"2022-01-05T12:33:19.614156Z","shell.execute_reply.started":"2022-01-05T12:33:19.609061Z","shell.execute_reply":"2022-01-05T12:33:19.613254Z"}}},{"cell_type":"code","source":"y_train_boxcox, best_lambda = stats.boxcox(y_train_log_new )\ndisplay(\"Skwness is :\", skew(y_train_boxcox),\n       \"Kurtosis is :\", kurtosis(y_train_boxcox))\nsns.distplot(y_train_boxcox, hist = True ,color='black', kde=True ,label=['distribution'])","metadata":{"execution":{"iopub.status.busy":"2022-01-05T14:16:10.830042Z","iopub.execute_input":"2022-01-05T14:16:10.830642Z","iopub.status.idle":"2022-01-05T14:17:14.939454Z","shell.execute_reply.started":"2022-01-05T14:16:10.830582Z","shell.execute_reply":"2022-01-05T14:17:14.938476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evulating the test ","metadata":{}},{"cell_type":"code","source":"y_test_new = y_test.copy(deep = True)+1\ndisplay(\"Skwness  befor   :\", skew(y_test_new),\n       \"Kurtosis  befor  :\", kurtosis(y_test_new))\ndisplay(\"Skwness  after   :\", skew(y_test_boxcox),\n       \"Kurtosis  befor  :\", kurtosis(y_test_boxcox))\ny_test_boxcox, best_lambda = stats.boxcox(y_test_new )\nsns.distplot(y_test_boxcox, hist = True ,color='black', kde=True ,label=['distribution'])","metadata":{"execution":{"iopub.status.busy":"2022-01-05T14:30:16.235104Z","iopub.execute_input":"2022-01-05T14:30:16.235471Z","iopub.status.idle":"2022-01-05T14:30:30.268028Z","shell.execute_reply.started":"2022-01-05T14:30:16.235433Z","shell.execute_reply":"2022-01-05T14:30:30.267262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### outlier removal","metadata":{}},{"cell_type":"code","source":"outlier_plotbox(y_test,removeOutliers(y_test))","metadata":{"execution":{"iopub.status.busy":"2022-01-05T14:54:39.439114Z","iopub.execute_input":"2022-01-05T14:54:39.439452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cleaning up\ndel building_metadata\ndel weather_train\nimport gc; gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.671049Z","iopub.status.idle":"2022-01-05T13:37:57.671556Z","shell.execute_reply.started":"2022-01-05T13:37:57.671362Z","shell.execute_reply":"2022-01-05T13:37:57.671391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dra","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.672531Z","iopub.status.idle":"2022-01-05T13:37:57.673023Z","shell.execute_reply.started":"2022-01-05T13:37:57.672804Z","shell.execute_reply":"2022-01-05T13:37:57.672832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"## Categorical data","metadata":{}},{"cell_type":"markdown","source":"#### 'primary_use' is the first categorical data that needs to be dealt with","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\n\nbuilding_count = train.primary_use.value_counts()\nprint('Number of Buildings by Primary Use:')\nplt.figure(figsize=(15,3))\nsns.barplot(x=building_count.index,y=train['primary_use'].value_counts())\nplt.xticks(rotation = 90)\nplt.ylabel('Number of Buildings')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import preprocessing\n\n## Label encoding isn't correct in this case as this data is not ordinal (Higher numbers aren't better)\n## can use get_dummies or one_hot_encoding only\n## Since all labels are present in both data sets (train+test) we can just use get_dummies\n\ntrain = pd.get_dummies(train, columns=['primary_use'])\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.675416Z","iopub.status.idle":"2022-01-05T13:37:57.6759Z","shell.execute_reply.started":"2022-01-05T13:37:57.675687Z","shell.execute_reply":"2022-01-05T13:37:57.675724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.676702Z","iopub.status.idle":"2022-01-05T13:37:57.677371Z","shell.execute_reply.started":"2022-01-05T13:37:57.67714Z","shell.execute_reply":"2022-01-05T13:37:57.677165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.678452Z","iopub.status.idle":"2022-01-05T13:37:57.678771Z","shell.execute_reply.started":"2022-01-05T13:37:57.678601Z","shell.execute_reply":"2022-01-05T13:37:57.678617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Convert 'timestamp' to datetime object","metadata":{}},{"cell_type":"code","source":"train['timestamp'] = pd.to_datetime(train.timestamp)","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.679725Z","iopub.status.idle":"2022-01-05T13:37:57.680019Z","shell.execute_reply.started":"2022-01-05T13:37:57.679864Z","shell.execute_reply":"2022-01-05T13:37:57.679881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate elapsed years for each building\ntrain['elapsed_years'] = train.timestamp.dt.year  - train.year_built","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.680805Z","iopub.status.idle":"2022-01-05T13:37:57.681088Z","shell.execute_reply.started":"2022-01-05T13:37:57.680937Z","shell.execute_reply":"2022-01-05T13:37:57.680952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Converting 'timestamp' to epoch time (n. of seconds since 1/1/1970) \nepoch time is ordinal in nature (more recent is higher) instead of datetime string which is meaningless when training","metadata":{}},{"cell_type":"code","source":"train['timestamp'] = pd.to_datetime(train['timestamp']).values.astype(np.int64) // (10 ** 10)","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.68181Z","iopub.status.idle":"2022-01-05T13:37:57.682103Z","shell.execute_reply.started":"2022-01-05T13:37:57.681945Z","shell.execute_reply":"2022-01-05T13:37:57.681961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.682872Z","iopub.status.idle":"2022-01-05T13:37:57.683156Z","shell.execute_reply.started":"2022-01-05T13:37:57.683004Z","shell.execute_reply":"2022-01-05T13:37:57.68302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Since, the site 0 electric meter readings are in kBTU and all other readings are in kWh, we will multiply by 0.2931","metadata":{}},{"cell_type":"code","source":"# To check before and after (Function is working correctly)\ntrain.loc[[45]]","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.68453Z","iopub.status.idle":"2022-01-05T13:37:57.684932Z","shell.execute_reply.started":"2022-01-05T13:37:57.68469Z","shell.execute_reply":"2022-01-05T13:37:57.684707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.tail()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.686142Z","iopub.status.idle":"2022-01-05T13:37:57.686483Z","shell.execute_reply.started":"2022-01-05T13:37:57.68632Z","shell.execute_reply":"2022-01-05T13:37:57.686338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.loc[train.site_id==0, 'meter_reading'] = train.loc[train.site_id==0, 'meter_reading'] * 0.2931","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.68752Z","iopub.status.idle":"2022-01-05T13:37:57.687821Z","shell.execute_reply.started":"2022-01-05T13:37:57.687658Z","shell.execute_reply":"2022-01-05T13:37:57.687674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.loc[[45]]","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.688665Z","iopub.status.idle":"2022-01-05T13:37:57.688963Z","shell.execute_reply.started":"2022-01-05T13:37:57.68881Z","shell.execute_reply":"2022-01-05T13:37:57.688826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.tail()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.689962Z","iopub.status.idle":"2022-01-05T13:37:57.690294Z","shell.execute_reply.started":"2022-01-05T13:37:57.690103Z","shell.execute_reply":"2022-01-05T13:37:57.690119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.691303Z","iopub.status.idle":"2022-01-05T13:37:57.691602Z","shell.execute_reply.started":"2022-01-05T13:37:57.691442Z","shell.execute_reply":"2022-01-05T13:37:57.691459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fixing outliers and skewness by observing Data histograms","metadata":{}},{"cell_type":"code","source":"plot_cols = [\"meter\",\"timestamp\",\"meter_reading\",\"square_feet\",\"year_built\",\"floor_count\",\n\"air_temperature\",\"cloud_coverage\",\"dew_temperature\",\"precip_depth_1_hr\", \"elapsed_years\"] ","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.692585Z","iopub.status.idle":"2022-01-05T13:37:57.692887Z","shell.execute_reply.started":"2022-01-05T13:37:57.69273Z","shell.execute_reply":"2022-01-05T13:37:57.692747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\ntrain.hist(bins=50, figsize=(20,15), column=plot_cols)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.694553Z","iopub.status.idle":"2022-01-05T13:37:57.694948Z","shell.execute_reply.started":"2022-01-05T13:37:57.694708Z","shell.execute_reply":"2022-01-05T13:37:57.694801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.695911Z","iopub.status.idle":"2022-01-05T13:37:57.696213Z","shell.execute_reply.started":"2022-01-05T13:37:57.696045Z","shell.execute_reply":"2022-01-05T13:37:57.69606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Finding outliers:\n* meter_reading: the 75th percentile is 2.50e2 while max is 2.19e7 (there is clear outlier(s))\n* precip_depth_1_hr: same as meter_reading\n\n### skewness:\nfrom the histograms:\n* square_feet is skewed\n* floor_count is skewed","metadata":{}},{"cell_type":"markdown","source":"### Removing outliers and fixing skewness","metadata":{}},{"cell_type":"code","source":"# Outliers\ntrain = train[train['meter_reading'] < 3.5e2 ] ## removes outliers and shrinks dataset for faster training\ntrain = train[train['precip_depth_1_hr'] < 180 ]\ntrain = train[train['precip_depth_1_hr'] >= 0 ] ## precipitation depth cannot be < 0","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.697813Z","iopub.status.idle":"2022-01-05T13:37:57.698102Z","shell.execute_reply.started":"2022-01-05T13:37:57.69795Z","shell.execute_reply":"2022-01-05T13:37:57.697965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.699122Z","iopub.status.idle":"2022-01-05T13:37:57.699454Z","shell.execute_reply.started":"2022-01-05T13:37:57.699295Z","shell.execute_reply":"2022-01-05T13:37:57.699313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Skewness\ntrain[\"meter_reading\"] = np.log1p(train['meter_reading'])\ntrain[\"square_feet\"] = np.log1p(train['square_feet'])\ntrain[\"precip_depth_1_hr\"] = np.log1p(train['precip_depth_1_hr'])\n\n# both square_feet and precip_depth_1_hr are unskewed for \n# train/test data (will be split later), so they can remain unchanged\n# BUT meter_reading (the target) should be reversed to its original state\n# after prediction (at the end), using: np.expm1(model.predict(x_test))","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.700271Z","iopub.status.idle":"2022-01-05T13:37:57.700566Z","shell.execute_reply.started":"2022-01-05T13:37:57.700406Z","shell.execute_reply":"2022-01-05T13:37:57.700422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\ntrain.hist(bins=50, figsize=(20,15), column=plot_cols)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.701537Z","iopub.status.idle":"2022-01-05T13:37:57.701911Z","shell.execute_reply.started":"2022-01-05T13:37:57.701673Z","shell.execute_reply":"2022-01-05T13:37:57.701689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.70275Z","iopub.status.idle":"2022-01-05T13:37:57.703029Z","shell.execute_reply.started":"2022-01-05T13:37:57.702879Z","shell.execute_reply":"2022-01-05T13:37:57.702895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.70385Z","iopub.status.idle":"2022-01-05T13:37:57.704128Z","shell.execute_reply.started":"2022-01-05T13:37:57.703979Z","shell.execute_reply":"2022-01-05T13:37:57.703994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train-Test split","metadata":{}},{"cell_type":"code","source":"## First we have to find x and y\n\ny = train['meter_reading'] # meter_reading is y\ntrain.drop(columns=[\"meter_reading\"], inplace=True) # train is x\n\ngc.collect()\n\n# splitting\nfrom sklearn.model_selection import train_test_split\n\nx_train,x_test,y_train,y_test = train_test_split(train,y,test_size=0.2)\n\ndel y; del train; gc.collect();","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.705023Z","iopub.status.idle":"2022-01-05T13:37:57.705325Z","shell.execute_reply.started":"2022-01-05T13:37:57.705158Z","shell.execute_reply":"2022-01-05T13:37:57.705174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Missing data replacement and data normalization","metadata":{}},{"cell_type":"code","source":"print(pd.isnull(y_train).any())\nprint(pd.isnull(y_test).any())","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.70617Z","iopub.status.idle":"2022-01-05T13:37:57.706496Z","shell.execute_reply.started":"2022-01-05T13:37:57.70634Z","shell.execute_reply":"2022-01-05T13:37:57.706356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since the target columns for both train and test sets contain no nulls, they will not be treated with simple imputer.","metadata":{}},{"cell_type":"code","source":"# how many nulls in each column before replacing\nprint(pd.isnull(x_train).sum())\nprint(\"------------\")\nprint(pd.isnull(x_test).sum())","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.707561Z","iopub.status.idle":"2022-01-05T13:37:57.707857Z","shell.execute_reply.started":"2022-01-05T13:37:57.707694Z","shell.execute_reply":"2022-01-05T13:37:57.707711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.impute import SimpleImputer # Imputation transformer for completing missing values.\nfrom sklearn.preprocessing import StandardScaler\n\n\n# Storing column and row labels of x_train, x_test to reconstruct it as a dataframe later\ncolumns_train = x_train.columns\nindex_train = x_train.index\ncolumns_test = x_test.columns\nindex_test = x_test.index\n\nscaler = StandardScaler()\nx_train = pd.DataFrame(scaler.fit_transform(x_train))\nx_test = pd.DataFrame(scaler.transform(x_test))\n\nx_train.columns = columns_train\nx_train.index = index_train\nx_test.columns = columns_test\nx_test.index = index_test\n\nimp_mean = SimpleImputer(missing_values = np.nan, strategy = 'mean') ## imputer that replaces with column mean\nx_train = pd.DataFrame(imp_mean.fit_transform(x_train))\nx_test = pd.DataFrame(imp_mean.transform(x_test)) # mean is only fitted (calculated) for test data\n\nx_train.columns = columns_train\nx_train.index = index_train\nx_test.columns = columns_test\nx_test.index = index_test\n\n# cleaning up variables\ndel columns_train; del index_train; \ndel columns_test; del index_test; \ndel imp_mean; gc.collect();","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.70883Z","iopub.status.idle":"2022-01-05T13:37:57.709117Z","shell.execute_reply.started":"2022-01-05T13:37:57.708966Z","shell.execute_reply":"2022-01-05T13:37:57.708981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# seeing if ifunction worked properly\nprint(x_train.isnull().any())\nprint(\"------------\")\nprint(x_test.isnull().any())","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.710465Z","iopub.status.idle":"2022-01-05T13:37:57.710792Z","shell.execute_reply.started":"2022-01-05T13:37:57.710618Z","shell.execute_reply":"2022-01-05T13:37:57.710641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train.sample(10)","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.713453Z","iopub.status.idle":"2022-01-05T13:37:57.713916Z","shell.execute_reply.started":"2022-01-05T13:37:57.713668Z","shell.execute_reply":"2022-01-05T13:37:57.713693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Features selection","metadata":{}},{"cell_type":"markdown","source":"### Correlation","metadata":{}},{"cell_type":"code","source":"corr_cols = ['meter', 'timestamp', 'square_feet', 'year_built',\n       'floor_count', 'air_temperature', 'cloud_coverage', 'dew_temperature',\n       'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction',\n       'wind_speed', 'elapsed_years'] # taking data that did not originate from primary_use categorical column","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.715374Z","iopub.status.idle":"2022-01-05T13:37:57.715824Z","shell.execute_reply.started":"2022-01-05T13:37:57.715578Z","shell.execute_reply":"2022-01-05T13:37:57.715603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corrMatrix = x_train[corr_cols].corr() ## find correlation between columns that did not originate from categorical data","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.71693Z","iopub.status.idle":"2022-01-05T13:37:57.717408Z","shell.execute_reply.started":"2022-01-05T13:37:57.717131Z","shell.execute_reply":"2022-01-05T13:37:57.717165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(12,12))  \nsns.heatmap(corrMatrix, annot=True, ax=ax)\nplt.show()\ndel fig; del ax; del corrMatrix; gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.718884Z","iopub.status.idle":"2022-01-05T13:37:57.719375Z","shell.execute_reply.started":"2022-01-05T13:37:57.719095Z","shell.execute_reply":"2022-01-05T13:37:57.71912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Eliminating highly correlated columns\n### dew_temp and air_temp are strongly positively correlated (corr=0.7)\n### one of them may be dropped","metadata":{}},{"cell_type":"code","source":"## We decided to drop dew_temp \nx_train.drop(columns=[\"dew_temperature\"], inplace=True)\nx_test.drop(columns=[\"dew_temperature\"], inplace=True)\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.720623Z","iopub.status.idle":"2022-01-05T13:37:57.721082Z","shell.execute_reply.started":"2022-01-05T13:37:57.720835Z","shell.execute_reply":"2022-01-05T13:37:57.72086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set(x_train.columns) == set(x_test.columns)","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:37:57.723031Z","iopub.status.idle":"2022-01-05T13:37:57.723521Z","shell.execute_reply.started":"2022-01-05T13:37:57.723255Z","shell.execute_reply":"2022-01-05T13:37:57.723282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}