{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport gc","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:14:44.314519Z","iopub.execute_input":"2022-01-21T15:14:44.314933Z","iopub.status.idle":"2022-01-21T15:14:45.349617Z","shell.execute_reply.started":"2022-01-21T15:14:44.314834Z","shell.execute_reply":"2022-01-21T15:14:45.348744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/ashrae-energy-prediction/train.csv')\nweather_train = pd.read_csv('../input/ashrae-energy-prediction/weather_train.csv')\nbuilding = pd.read_csv('../input/ashrae-energy-prediction/building_metadata.csv')\nweather_test = pd.read_csv('../input/ashrae-energy-prediction/weather_test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:14:45.351119Z","iopub.execute_input":"2022-01-21T15:14:45.351463Z","iopub.status.idle":"2022-01-21T15:15:01.574980Z","shell.execute_reply.started":"2022-01-21T15:14:45.351431Z","shell.execute_reply":"2022-01-21T15:15:01.574168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Function to reduce the DF size\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:01.577156Z","iopub.execute_input":"2022-01-21T15:15:01.577377Z","iopub.status.idle":"2022-01-21T15:15:01.591960Z","shell.execute_reply.started":"2022-01-21T15:15:01.577351Z","shell.execute_reply":"2022-01-21T15:15:01.591027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = reduce_mem_usage(train)\nweather_train = reduce_mem_usage(weather_train)\nbuilding = reduce_mem_usage(building)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:01.593898Z","iopub.execute_input":"2022-01-21T15:15:01.594257Z","iopub.status.idle":"2022-01-21T15:15:01.989593Z","shell.execute_reply.started":"2022-01-21T15:15:01.594223Z","shell.execute_reply":"2022-01-21T15:15:01.988761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_test = reduce_mem_usage(weather_test)\nbuilding = reduce_mem_usage(building) ","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:01.990834Z","iopub.execute_input":"2022-01-21T15:15:01.991104Z","iopub.status.idle":"2022-01-21T15:15:02.040782Z","shell.execute_reply.started":"2022-01-21T15:15:01.991074Z","shell.execute_reply":"2022-01-21T15:15:02.039912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:02.042068Z","iopub.execute_input":"2022-01-21T15:15:02.042918Z","iopub.status.idle":"2022-01-21T15:15:02.061036Z","shell.execute_reply.started":"2022-01-21T15:15:02.042875Z","shell.execute_reply":"2022-01-21T15:15:02.060140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:02.062621Z","iopub.execute_input":"2022-01-21T15:15:02.062951Z","iopub.status.idle":"2022-01-21T15:15:02.078987Z","shell.execute_reply.started":"2022-01-21T15:15:02.062914Z","shell.execute_reply":"2022-01-21T15:15:02.078139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['timestamp'] = pd.to_datetime(train['timestamp'])\nprint(f'''{train['timestamp'].dtype}\n{train['timestamp'].min()}\n{train['timestamp'].max()}''')","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:02.080257Z","iopub.execute_input":"2022-01-21T15:15:02.080452Z","iopub.status.idle":"2022-01-21T15:15:05.766370Z","shell.execute_reply.started":"2022-01-21T15:15:02.080428Z","shell.execute_reply":"2022-01-21T15:15:05.765477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:05.768087Z","iopub.execute_input":"2022-01-21T15:15:05.768328Z","iopub.status.idle":"2022-01-21T15:15:05.904663Z","shell.execute_reply.started":"2022-01-21T15:15:05.768297Z","shell.execute_reply":"2022-01-21T15:15:05.903712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['meter_reading'].describe()\n","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:05.908860Z","iopub.execute_input":"2022-01-21T15:15:05.909095Z","iopub.status.idle":"2022-01-21T15:15:06.641435Z","shell.execute_reply.started":"2022-01-21T15:15:05.909061Z","shell.execute_reply":"2022-01-21T15:15:06.640651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculating the zero percent consuming electricity","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:06.643853Z","iopub.execute_input":"2022-01-21T15:15:06.644067Z","iopub.status.idle":"2022-01-21T15:15:06.648300Z","shell.execute_reply.started":"2022-01-21T15:15:06.644039Z","shell.execute_reply":"2022-01-21T15:15:06.647434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Percent of zero read values:', \"%.2f\"%(train[train['meter_reading']== 0].shape[0] /train.shape[0]))\ntrain[train['meter_reading'] == 0]","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:06.649406Z","iopub.execute_input":"2022-01-21T15:15:06.649595Z","iopub.status.idle":"2022-01-21T15:15:06.967445Z","shell.execute_reply.started":"2022-01-21T15:15:06.649571Z","shell.execute_reply":"2022-01-21T15:15:06.966768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculating the percent of the zero consuming electricity buildings:","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:06.968335Z","iopub.execute_input":"2022-01-21T15:15:06.969145Z","iopub.status.idle":"2022-01-21T15:15:06.972618Z","shell.execute_reply.started":"2022-01-21T15:15:06.969112Z","shell.execute_reply":"2022-01-21T15:15:06.971851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"\"\"\"\nTotal number of buildings: {train['building_id'].nunique()}\nNumber of buildings with zero readings: {train[train['meter_reading']== 0][\"building_id\"].value_counts().shape[0]}\nPercentage: {round(train[train['meter_reading']== 0][\"building_id\"].value_counts().shape[0] / train['building_id'].nunique(),2)}\n\nNumber of zero readings per building: \n{train[train['meter_reading']== 0][\"building_id\"].value_counts().sort_values(ascending = False)}\n\n\n\"\"\"\n     )\n","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:06.973753Z","iopub.execute_input":"2022-01-21T15:15:06.973974Z","iopub.status.idle":"2022-01-21T15:15:07.630055Z","shell.execute_reply.started":"2022-01-21T15:15:06.973946Z","shell.execute_reply":"2022-01-21T15:15:07.629184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(train[train['meter_reading']== 0]).groupby('building_id')['meter'].value_counts().sort_values(ascending = False)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:07.632598Z","iopub.execute_input":"2022-01-21T15:15:07.633298Z","iopub.status.idle":"2022-01-21T15:15:08.072122Z","shell.execute_reply.started":"2022-01-21T15:15:07.633262Z","shell.execute_reply":"2022-01-21T15:15:08.071204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of zero buliding for each buliding","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:08.073086Z","iopub.execute_input":"2022-01-21T15:15:08.073368Z","iopub.status.idle":"2022-01-21T15:15:08.076818Z","shell.execute_reply.started":"2022-01-21T15:15:08.073336Z","shell.execute_reply":"2022-01-21T15:15:08.076051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number of zero readings for each meter:\\n',train[train['meter_reading']== 0][\"meter\"].value_counts(),'\\n')\n\nfor i in range(train[\"meter\"].nunique()):\n    percent = round(train[train['meter_reading']== 0][\"meter\"].value_counts()[i] /train[\"meter\"].value_counts()[i],2)\n    print(f'% of zero reads for Meter {i}: {percent}')","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:08.077854Z","iopub.execute_input":"2022-01-21T15:15:08.078159Z","iopub.status.idle":"2022-01-21T15:15:09.207925Z","shell.execute_reply.started":"2022-01-21T15:15:08.078133Z","shell.execute_reply":"2022-01-21T15:15:09.205742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining each parameter ","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:09.209566Z","iopub.execute_input":"2022-01-21T15:15:09.209918Z","iopub.status.idle":"2022-01-21T15:15:09.213936Z","shell.execute_reply.started":"2022-01-21T15:15:09.209875Z","shell.execute_reply":"2022-01-21T15:15:09.213112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['meter'] = pd.Categorical(train['meter']).rename_categories({0: 'electricity', 1: 'chilledwater', 2: 'steam', 3: 'hotwater'})","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:09.214927Z","iopub.execute_input":"2022-01-21T15:15:09.215122Z","iopub.status.idle":"2022-01-21T15:15:09.548225Z","shell.execute_reply.started":"2022-01-21T15:15:09.215098Z","shell.execute_reply":"2022-01-21T15:15:09.547306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g = sns.FacetGrid(train[train['meter_reading']== 0], col=\"meter\",hue = 'meter',palette='coolwarm',col_wrap=2,height=3, aspect=2)\ng.map(sns.histplot, 'timestamp', bins=12)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:09.549525Z","iopub.execute_input":"2022-01-21T15:15:09.549768Z","iopub.status.idle":"2022-01-21T15:15:22.867663Z","shell.execute_reply.started":"2022-01-21T15:15:09.549741Z","shell.execute_reply":"2022-01-21T15:15:22.866846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 1, figsize=(14, 6))\nsns.boxplot(y='meter', x='meter_reading', data=train, showfliers=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:22.868817Z","iopub.execute_input":"2022-01-21T15:15:22.869025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 1, figsize=(14, 6))\nsns.boxplot(y='meter', x='meter_reading', data=train[train['meter_reading'] !=  0 ], showfliers=False);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(data=train.groupby(['timestamp']).sum(), x=\"timestamp\", y=\"meter_reading\") ","metadata":{"execution":{"iopub.status.idle":"2022-01-21T15:15:36.509550Z","shell.execute_reply.started":"2022-01-21T15:15:34.910712Z","shell.execute_reply":"2022-01-21T15:15:36.508734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.relplot(data=(train[train['meter'].isin(['electricity','chilledwater','hotwater'])].groupby(['timestamp','meter']).sum()), x=\"timestamp\", y=\"meter_reading\",col=\"meter\", hue=\"meter\",kind=\"line\")","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:36.511734Z","iopub.execute_input":"2022-01-21T15:15:36.511969Z","iopub.status.idle":"2022-01-21T15:15:43.675576Z","shell.execute_reply.started":"2022-01-21T15:15:36.511941Z","shell.execute_reply":"2022-01-21T15:15:43.674602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:43.676876Z","iopub.execute_input":"2022-01-21T15:15:43.677130Z","iopub.status.idle":"2022-01-21T15:15:43.815416Z","shell.execute_reply.started":"2022-01-21T15:15:43.677101Z","shell.execute_reply":"2022-01-21T15:15:43.814458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data Analysis in Buliding MetaData:","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:43.817291Z","iopub.execute_input":"2022-01-21T15:15:43.817710Z","iopub.status.idle":"2022-01-21T15:15:43.824365Z","shell.execute_reply.started":"2022-01-21T15:15:43.817646Z","shell.execute_reply":"2022-01-21T15:15:43.823605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"building.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:43.825703Z","iopub.execute_input":"2022-01-21T15:15:43.826295Z","iopub.status.idle":"2022-01-21T15:15:43.842844Z","shell.execute_reply.started":"2022-01-21T15:15:43.826261Z","shell.execute_reply":"2022-01-21T15:15:43.841991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"building.info()\n","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:43.848754Z","iopub.execute_input":"2022-01-21T15:15:43.849629Z","iopub.status.idle":"2022-01-21T15:15:43.864354Z","shell.execute_reply.started":"2022-01-21T15:15:43.849593Z","shell.execute_reply":"2022-01-21T15:15:43.863456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"building.describe()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:43.866374Z","iopub.execute_input":"2022-01-21T15:15:43.866617Z","iopub.status.idle":"2022-01-21T15:15:43.897324Z","shell.execute_reply.started":"2022-01-21T15:15:43.866589Z","shell.execute_reply":"2022-01-21T15:15:43.896381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"building.describe(include = 'O')","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:43.898757Z","iopub.execute_input":"2022-01-21T15:15:43.899509Z","iopub.status.idle":"2022-01-21T15:15:43.914025Z","shell.execute_reply.started":"2022-01-21T15:15:43.899471Z","shell.execute_reply":"2022-01-21T15:15:43.913117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Null value counts:\\n{building.isnull().sum()}\\n')\nfor col in list(building.columns):\n    if building[col].isnull().sum() > 0:\n        print(f'% of null in column {col}: {round(building[col].isnull().sum() / building.shape[0], 2 )}' )","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:43.915131Z","iopub.execute_input":"2022-01-21T15:15:43.915354Z","iopub.status.idle":"2022-01-21T15:15:43.928988Z","shell.execute_reply.started":"2022-01-21T15:15:43.915327Z","shell.execute_reply":"2022-01-21T15:15:43.928365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"building.fillna('Missing', inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:43.930313Z","iopub.execute_input":"2022-01-21T15:15:43.930803Z","iopub.status.idle":"2022-01-21T15:15:43.936037Z","shell.execute_reply.started":"2022-01-21T15:15:43.930771Z","shell.execute_reply":"2022-01-21T15:15:43.935250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('For Primary Usage Areas\\n')\nprint('% of Null year_built Values Less Than 50%')\nfor usage in list(building['primary_use'].unique()):\n    percent  = round(sum(building[building['primary_use']== usage]['year_built']== 'Missing') / building[building['primary_use']== usage].shape[0], 2)\n    if percent < 0.5:\n        print(f'{usage}: {percent}')\n        \nprint('\\n% of Null year_built Values Higher Than 50%')\nfor usage in list(building['primary_use'].unique()):\n    percent  = round(sum(building[building['primary_use']== usage]['year_built']== 'Missing') / building[building['primary_use']== usage].shape[0], 2)\n    if percent > 0.5:\n        print(f'{usage}: {percent}')","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:43.937172Z","iopub.execute_input":"2022-01-21T15:15:43.937400Z","iopub.status.idle":"2022-01-21T15:15:44.009258Z","shell.execute_reply.started":"2022-01-21T15:15:43.937373Z","shell.execute_reply":"2022-01-21T15:15:44.008414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('For Primary Usage Areas\\n')\nprint('\\n% of Null floor_count Values Less Than 50%\\n')\nfor usage in list(building['primary_use'].unique()):\n    percent  = round(sum(building[building['primary_use']== usage]['floor_count']== 'Missing') / building[building['primary_use']== usage].shape[0], 2)\n    if percent < 0.5:\n        print(f'{usage}: {percent}')\n        \nprint('\\n% of Null floor_count Values Higher Than 50%\\n')\nfor usage in list(building['primary_use'].unique()):\n    percent  = round(sum(building[building['primary_use']== usage]['floor_count']== 'Missing') / building[building['primary_use']== usage].shape[0], 2)\n    if percent > 0.5:\n        print(f'{usage}: {percent}')","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.010790Z","iopub.execute_input":"2022-01-21T15:15:44.011211Z","iopub.status.idle":"2022-01-21T15:15:44.076845Z","shell.execute_reply.started":"2022-01-21T15:15:44.011170Z","shell.execute_reply":"2022-01-21T15:15:44.075744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dropping floor count and year built:","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.078433Z","iopub.execute_input":"2022-01-21T15:15:44.078893Z","iopub.status.idle":"2022-01-21T15:15:44.082494Z","shell.execute_reply.started":"2022-01-21T15:15:44.078850Z","shell.execute_reply":"2022-01-21T15:15:44.081621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"building.drop(['floor_count','year_built'],axis=1,inplace=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.083791Z","iopub.execute_input":"2022-01-21T15:15:44.084521Z","iopub.status.idle":"2022-01-21T15:15:44.096872Z","shell.execute_reply.started":"2022-01-21T15:15:44.084485Z","shell.execute_reply":"2022-01-21T15:15:44.096089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.098116Z","iopub.execute_input":"2022-01-21T15:15:44.099031Z","iopub.status.idle":"2022-01-21T15:15:44.243113Z","shell.execute_reply.started":"2022-01-21T15:15:44.098986Z","shell.execute_reply":"2022-01-21T15:15:44.242039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Testing weather data:","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.244420Z","iopub.execute_input":"2022-01-21T15:15:44.244659Z","iopub.status.idle":"2022-01-21T15:15:44.253020Z","shell.execute_reply.started":"2022-01-21T15:15:44.244628Z","shell.execute_reply":"2022-01-21T15:15:44.252222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train.head()\n","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.254416Z","iopub.execute_input":"2022-01-21T15:15:44.254878Z","iopub.status.idle":"2022-01-21T15:15:44.276480Z","shell.execute_reply.started":"2022-01-21T15:15:44.254831Z","shell.execute_reply":"2022-01-21T15:15:44.275693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train.info()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.277937Z","iopub.execute_input":"2022-01-21T15:15:44.278336Z","iopub.status.idle":"2022-01-21T15:15:44.320407Z","shell.execute_reply.started":"2022-01-21T15:15:44.278302Z","shell.execute_reply":"2022-01-21T15:15:44.319761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculating the timestamp:","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.321531Z","iopub.execute_input":"2022-01-21T15:15:44.321893Z","iopub.status.idle":"2022-01-21T15:15:44.325510Z","shell.execute_reply.started":"2022-01-21T15:15:44.321862Z","shell.execute_reply":"2022-01-21T15:15:44.324293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train['timestamp'] = pd.to_datetime(weather_train['timestamp'])\nprint(f'''{weather_train['timestamp'].dtype}\n{weather_train['timestamp'].min()}\n{weather_train['timestamp'].max()}''')","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.326928Z","iopub.execute_input":"2022-01-21T15:15:44.327817Z","iopub.status.idle":"2022-01-21T15:15:44.386717Z","shell.execute_reply.started":"2022-01-21T15:15:44.327770Z","shell.execute_reply":"2022-01-21T15:15:44.385634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Null value check","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.388205Z","iopub.execute_input":"2022-01-21T15:15:44.388816Z","iopub.status.idle":"2022-01-21T15:15:44.393989Z","shell.execute_reply.started":"2022-01-21T15:15:44.388780Z","shell.execute_reply":"2022-01-21T15:15:44.393229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Null value counts:\\n{weather_train.isnull().sum()}\\n')\nfor col in list(weather_train.columns):\n    if weather_train[col].isnull().sum() > 0:\n        print(f'% of null in column {col}: {round(weather_train[col].isnull().sum() / weather_train.shape[0], 4 )}' )","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.395508Z","iopub.execute_input":"2022-01-21T15:15:44.396040Z","iopub.status.idle":"2022-01-21T15:15:44.427288Z","shell.execute_reply.started":"2022-01-21T15:15:44.396002Z","shell.execute_reply":"2022-01-21T15:15:44.426385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train.drop(['cloud_coverage','precip_depth_1_hr'], axis = 1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.428985Z","iopub.execute_input":"2022-01-21T15:15:44.429316Z","iopub.status.idle":"2022-01-21T15:15:44.439075Z","shell.execute_reply.started":"2022-01-21T15:15:44.429272Z","shell.execute_reply":"2022-01-21T15:15:44.437934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train['hour'] = weather_train.timestamp.dt.hour\nweather_train['month'] = weather_train.timestamp.dt.month","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.440422Z","iopub.execute_input":"2022-01-21T15:15:44.440944Z","iopub.status.idle":"2022-01-21T15:15:44.479626Z","shell.execute_reply.started":"2022-01-21T15:15:44.440904Z","shell.execute_reply":"2022-01-21T15:15:44.478741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using mean values to impute meteorological data:","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.480772Z","iopub.execute_input":"2022-01-21T15:15:44.481207Z","iopub.status.idle":"2022-01-21T15:15:44.484658Z","shell.execute_reply.started":"2022-01-21T15:15:44.481172Z","shell.execute_reply":"2022-01-21T15:15:44.483971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def site_mean_weather(table):\n    for col in list(table.columns[table.isnull().any()]):\n        imputaion = table.groupby(['site_id','hour','month'])[col].transform('mean')\n        table[col].fillna(imputaion,inplace = True)\n    print('Imputation with mean values is completed.')\n     \nsite_mean_weather(weather_train)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.485636Z","iopub.execute_input":"2022-01-21T15:15:44.486399Z","iopub.status.idle":"2022-01-21T15:15:44.561895Z","shell.execute_reply.started":"2022-01-21T15:15:44.486350Z","shell.execute_reply":"2022-01-21T15:15:44.560947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.563279Z","iopub.execute_input":"2022-01-21T15:15:44.563570Z","iopub.status.idle":"2022-01-21T15:15:44.577022Z","shell.execute_reply.started":"2022-01-21T15:15:44.563529Z","shell.execute_reply":"2022-01-21T15:15:44.576123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# After imputation I realize there are still missing values in sea_level_pressure column, so I droped that column to.","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.578150Z","iopub.execute_input":"2022-01-21T15:15:44.578381Z","iopub.status.idle":"2022-01-21T15:15:44.582649Z","shell.execute_reply.started":"2022-01-21T15:15:44.578352Z","shell.execute_reply":"2022-01-21T15:15:44.581836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train.drop(['sea_level_pressure'], axis = 1, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.584422Z","iopub.execute_input":"2022-01-21T15:15:44.584848Z","iopub.status.idle":"2022-01-21T15:15:44.600533Z","shell.execute_reply.started":"2022-01-21T15:15:44.584805Z","shell.execute_reply":"2022-01-21T15:15:44.599663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.601838Z","iopub.execute_input":"2022-01-21T15:15:44.603055Z","iopub.status.idle":"2022-01-21T15:15:44.745547Z","shell.execute_reply.started":"2022-01-21T15:15:44.603013Z","shell.execute_reply":"2022-01-21T15:15:44.744602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data Analysis","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.746813Z","iopub.execute_input":"2022-01-21T15:15:44.747068Z","iopub.status.idle":"2022-01-21T15:15:44.756719Z","shell.execute_reply.started":"2022-01-21T15:15:44.747030Z","shell.execute_reply":"2022-01-21T15:15:44.755594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# To evaluate all train data together train, building and weather tables are merged together and unnecessary tables are deleted.","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.759463Z","iopub.execute_input":"2022-01-21T15:15:44.759879Z","iopub.status.idle":"2022-01-21T15:15:44.766710Z","shell.execute_reply.started":"2022-01-21T15:15:44.759833Z","shell.execute_reply":"2022-01-21T15:15:44.765975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.merge(train,building, on=\"building_id\", how=\"left\")\ndf = df.merge(weather_train, on=[\"site_id\", \"timestamp\"], how=\"left\")","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:44.768083Z","iopub.execute_input":"2022-01-21T15:15:44.768586Z","iopub.status.idle":"2022-01-21T15:15:52.156279Z","shell.execute_reply.started":"2022-01-21T15:15:44.768543Z","shell.execute_reply":"2022-01-21T15:15:52.155404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:52.157475Z","iopub.execute_input":"2022-01-21T15:15:52.157738Z","iopub.status.idle":"2022-01-21T15:15:52.163793Z","shell.execute_reply.started":"2022-01-21T15:15:52.157707Z","shell.execute_reply":"2022-01-21T15:15:52.162911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:52.165099Z","iopub.execute_input":"2022-01-21T15:15:52.165317Z","iopub.status.idle":"2022-01-21T15:15:52.191306Z","shell.execute_reply.started":"2022-01-21T15:15:52.165291Z","shell.execute_reply":"2022-01-21T15:15:52.190666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.nunique()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:52.192554Z","iopub.execute_input":"2022-01-21T15:15:52.192788Z","iopub.status.idle":"2022-01-21T15:15:57.050598Z","shell.execute_reply.started":"2022-01-21T15:15:52.192761Z","shell.execute_reply":"2022-01-21T15:15:57.049858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:57.051764Z","iopub.execute_input":"2022-01-21T15:15:57.051998Z","iopub.status.idle":"2022-01-21T15:15:59.766566Z","shell.execute_reply.started":"2022-01-21T15:15:57.051969Z","shell.execute_reply":"2022-01-21T15:15:59.765849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['month'] = df.timestamp.dt.month\ndf['hour'] = df.timestamp.dt.hour","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:15:59.767863Z","iopub.execute_input":"2022-01-21T15:15:59.768065Z","iopub.status.idle":"2022-01-21T15:16:03.524814Z","shell.execute_reply.started":"2022-01-21T15:15:59.768041Z","shell.execute_reply":"2022-01-21T15:16:03.524038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"site_mean_weather(df)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:16:03.526115Z","iopub.execute_input":"2022-01-21T15:16:03.526353Z","iopub.status.idle":"2022-01-21T15:16:12.774879Z","shell.execute_reply.started":"2022-01-21T15:16:03.526324Z","shell.execute_reply":"2022-01-21T15:16:12.773841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in list(df.columns[df.isnull().any()]):    \n    imputaion = df.groupby(['hour','month'])[col].transform('mean')\n    df[col].fillna(imputaion,inplace = True)\nprint('Imputation is completed.')","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:16:12.776167Z","iopub.execute_input":"2022-01-21T15:16:12.776380Z","iopub.status.idle":"2022-01-21T15:16:15.185593Z","shell.execute_reply.started":"2022-01-21T15:16:12.776352Z","shell.execute_reply":"2022-01-21T15:16:15.184757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train\ndel building\ndel weather_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:16:15.187165Z","iopub.execute_input":"2022-01-21T15:16:15.187634Z","iopub.status.idle":"2022-01-21T15:16:15.326611Z","shell.execute_reply.started":"2022-01-21T15:16:15.187591Z","shell.execute_reply":"2022-01-21T15:16:15.325895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:16:15.328069Z","iopub.execute_input":"2022-01-21T15:16:15.328412Z","iopub.status.idle":"2022-01-21T15:16:15.340466Z","shell.execute_reply.started":"2022-01-21T15:16:15.328370Z","shell.execute_reply":"2022-01-21T15:16:15.339509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[['primary_use','hour','month','site_id','building_id','wind_direction']] = df[['primary_use','hour','month','site_id','building_id','wind_direction']].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:16:15.342846Z","iopub.execute_input":"2022-01-21T15:16:15.343166Z","iopub.status.idle":"2022-01-21T15:16:20.465850Z","shell.execute_reply.started":"2022-01-21T15:16:15.343123Z","shell.execute_reply":"2022-01-21T15:16:20.464943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:16:20.467067Z","iopub.execute_input":"2022-01-21T15:16:20.467301Z","iopub.status.idle":"2022-01-21T15:16:20.604058Z","shell.execute_reply.started":"2022-01-21T15:16:20.467272Z","shell.execute_reply":"2022-01-21T15:16:20.603524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['cons/sqft'] = df['meter_reading'] / df['square_feet']","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:16:20.605269Z","iopub.execute_input":"2022-01-21T15:16:20.605644Z","iopub.status.idle":"2022-01-21T15:16:20.673528Z","shell.execute_reply.started":"2022-01-21T15:16:20.605613Z","shell.execute_reply":"2022-01-21T15:16:20.672909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.loc[(df['site_id'] == 0) & (df['meter'] == 'electricity'), 'meter_reading'] = df[(df['site_id'] == 0) & (df['meter'] == 'electricity')]['meter_reading'].apply(lambda x: x* 0.2931 )","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:16:20.674542Z","iopub.execute_input":"2022-01-21T15:16:20.674819Z","iopub.status.idle":"2022-01-21T15:16:21.428803Z","shell.execute_reply.started":"2022-01-21T15:16:20.674787Z","shell.execute_reply":"2022-01-21T15:16:21.427854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['day'] = df.timestamp.dt.year","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:16:21.430127Z","iopub.execute_input":"2022-01-21T15:16:21.430385Z","iopub.status.idle":"2022-01-21T15:16:23.281876Z","shell.execute_reply.started":"2022-01-21T15:16:21.430354Z","shell.execute_reply":"2022-01-21T15:16:23.280893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = reduce_mem_usage(df)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:16:23.283486Z","iopub.execute_input":"2022-01-21T15:16:23.283822Z","iopub.status.idle":"2022-01-21T15:16:25.164142Z","shell.execute_reply.started":"2022-01-21T15:16:23.283779Z","shell.execute_reply":"2022-01-21T15:16:25.163264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Analysis","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:16:25.165858Z","iopub.execute_input":"2022-01-21T15:16:25.166169Z","iopub.status.idle":"2022-01-21T15:16:25.170516Z","shell.execute_reply.started":"2022-01-21T15:16:25.166126Z","shell.execute_reply":"2022-01-21T15:16:25.169755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(12, 4),constrained_layout=True)\nfig.suptitle('Building Counts')\nsns.barplot(ax=axes[0],y=\"site_id\", x='building_id', data=df.groupby(['site_id'])['building_id'].nunique().reset_index())\naxes[0].set(xlabel = 'Building Count', ylabel = 'Site id')  \nsns.barplot(ax=axes[1],y=\"primary_use\", x='building_id', data=df.groupby(['primary_use'])['building_id'].nunique().reset_index())\naxes[1].set(xlabel = 'Building Count', ylabel = 'Primary Use')","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:16:25.171589Z","iopub.execute_input":"2022-01-21T15:16:25.171835Z","iopub.status.idle":"2022-01-21T15:16:31.449347Z","shell.execute_reply.started":"2022-01-21T15:16:25.171806Z","shell.execute_reply":"2022-01-21T15:16:31.448423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:16:31.450730Z","iopub.execute_input":"2022-01-21T15:16:31.451053Z","iopub.status.idle":"2022-01-21T15:16:31.595046Z","shell.execute_reply.started":"2022-01-21T15:16:31.451010Z","shell.execute_reply":"2022-01-21T15:16:31.594115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(2, 2, figsize=(20, 16),constrained_layout=True)\nfig.suptitle('Energy Consumption for Primary Use')\nsns.boxplot(ax=axes[0, 0], y='primary_use', x='meter_reading', data=df, showfliers=True)\nsns.boxplot(ax=axes[0, 1], y='primary_use', x='meter_reading', data=df, showfliers=False)\nsns.boxplot(ax=axes[1, 0], y='primary_use', x='cons/sqft', data=df, showfliers=True)\nsns.boxplot(ax=axes[1, 1], y='primary_use', x='cons/sqft', data=df, showfliers=False);","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:16:31.596804Z","iopub.execute_input":"2022-01-21T15:16:31.597123Z","iopub.status.idle":"2022-01-21T15:16:59.782711Z","shell.execute_reply.started":"2022-01-21T15:16:31.597081Z","shell.execute_reply":"2022-01-21T15:16:59.781798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(2, 2, figsize=(14, 16),constrained_layout=True)\nfig.suptitle('Energy Consumption for Sites')\nsns.boxplot(ax=axes[0, 0], y='site_id', x='meter_reading', data=df, showfliers=True)\nsns.boxplot(ax=axes[0, 1], y='site_id', x='meter_reading', data=df, showfliers=False)\nsns.boxplot(ax=axes[1, 0], y='site_id', x='cons/sqft', data=df, showfliers=True)\nsns.boxplot(ax=axes[1, 1], y='site_id', x='cons/sqft', data=df, showfliers=False);","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:16:59.783970Z","iopub.execute_input":"2022-01-21T15:16:59.784225Z","iopub.status.idle":"2022-01-21T15:17:25.268846Z","shell.execute_reply.started":"2022-01-21T15:16:59.784194Z","shell.execute_reply":"2022-01-21T15:17:25.268168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:17:25.269843Z","iopub.execute_input":"2022-01-21T15:17:25.270388Z","iopub.status.idle":"2022-01-21T15:17:25.460903Z","shell.execute_reply.started":"2022-01-21T15:17:25.270357Z","shell.execute_reply":"2022-01-21T15:17:25.460043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['day'] = df.timestamp.dt.day","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:17:25.462277Z","iopub.execute_input":"2022-01-21T15:17:25.462794Z","iopub.status.idle":"2022-01-21T15:17:27.311449Z","shell.execute_reply.started":"2022-01-21T15:17:25.462732Z","shell.execute_reply":"2022-01-21T15:17:27.310563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df['meter']=='electricity'].groupby(['site_id','month','day','building_id'])['meter_reading'].sum().sort_values(ascending = False).reset_index().head(100)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:17:27.313072Z","iopub.execute_input":"2022-01-21T15:17:27.313406Z","iopub.status.idle":"2022-01-21T15:18:02.167751Z","shell.execute_reply.started":"2022-01-21T15:17:27.313361Z","shell.execute_reply":"2022-01-21T15:18:02.166914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[df['meter']=='electricity'].groupby(['site_id','month','day','building_id'])['cons/sqft'].sum().sort_values(ascending = False).reset_index().head(100)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:18:02.168913Z","iopub.execute_input":"2022-01-21T15:18:02.169158Z","iopub.status.idle":"2022-01-21T15:18:36.128274Z","shell.execute_reply.started":"2022-01-21T15:18:02.169128Z","shell.execute_reply":"2022-01-21T15:18:36.127461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby(['site_id'])['meter_reading'].sum().sort_values(ascending = False).reset_index()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:18:36.129653Z","iopub.execute_input":"2022-01-21T15:18:36.129976Z","iopub.status.idle":"2022-01-21T15:18:36.536326Z","shell.execute_reply.started":"2022-01-21T15:18:36.129934Z","shell.execute_reply":"2022-01-21T15:18:36.535390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:18:36.537426Z","iopub.execute_input":"2022-01-21T15:18:36.537645Z","iopub.status.idle":"2022-01-21T15:18:36.697857Z","shell.execute_reply.started":"2022-01-21T15:18:36.537617Z","shell.execute_reply":"2022-01-21T15:18:36.696894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Removing Outliers","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:18:36.705181Z","iopub.execute_input":"2022-01-21T15:18:36.705658Z","iopub.status.idle":"2022-01-21T15:18:36.711273Z","shell.execute_reply.started":"2022-01-21T15:18:36.705621Z","shell.execute_reply":"2022-01-21T15:18:36.710697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total = 0\nprint('Outlier distribution in meter types')\nfor col in list(df['meter'].unique()):\n    r = np.percentile(df[df['meter'] == col]['cons/sqft'],75) + 1.5 * (np.percentile(df[df['meter'] == col]['cons/sqft'],75) - np.percentile(df[df['meter'] == col]['cons/sqft'],25))\n    print(f'''Percentage of outliers in {col} to all readings: { round(df[(df['meter'] == col) & (df['cons/sqft'] > r)].shape[0]/ df.shape[0],5)}''')\n    total +=  df[(df['meter'] == col) & (df['cons/sqft'] > r)].shape[0]\nprint(f'Total fraction of outliers {round(total / df.shape[0],5)}')\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:18:36.712407Z","iopub.execute_input":"2022-01-21T15:18:36.712826Z","iopub.status.idle":"2022-01-21T15:18:45.860575Z","shell.execute_reply.started":"2022-01-21T15:18:36.712790Z","shell.execute_reply":"2022-01-21T15:18:45.859747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total = 0\nprint('Outlier distribution in building types')\nfor col in list(df['primary_use'].unique()):\n    r = np.percentile(df[df['primary_use'] == col]['cons/sqft'],75) + 1.5 * (np.percentile(df[df['primary_use'] == col]['cons/sqft'],75) - np.percentile(df[df['primary_use'] == col]['cons/sqft'],25))\n    print(f'''Percentage of outliers in {col} to all readings: { round(df[(df['primary_use'] == col) & (df['cons/sqft'] > r)].shape[0]/ df.shape[0],5)}''')\n    total +=  df[(df['primary_use'] == col) & (df['cons/sqft'] > r)].shape[0]\nprint(f'Total fraction of outliers {round(total / df.shape[0],5)}')\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:18:45.862107Z","iopub.execute_input":"2022-01-21T15:18:45.862427Z","iopub.status.idle":"2022-01-21T15:19:01.335880Z","shell.execute_reply.started":"2022-01-21T15:18:45.862383Z","shell.execute_reply":"2022-01-21T15:19:01.335299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total = 0\nprint('Outlier distribution in building types')\nfor col in list(df['site_id'].unique()):\n    r = np.percentile(df[df['site_id'] == col]['cons/sqft'],75) + 1.5 * (np.percentile(df[df['site_id'] == col]['cons/sqft'],75) - np.percentile(df[df['site_id'] == col]['cons/sqft'],25))\n    print(f'''Percentage of outliers in {col} to all readings: { round(df[(df['site_id'] == col) & (df['cons/sqft'] > r)].shape[0]/ df.shape[0],5)}''')\n    total +=  df[(df['site_id'] == col) & (df['cons/sqft'] > r)].shape[0]\nprint(f'Total fraction of outliers {round(total / df.shape[0],5)}')\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:19:01.336969Z","iopub.execute_input":"2022-01-21T15:19:01.337177Z","iopub.status.idle":"2022-01-21T15:19:15.091494Z","shell.execute_reply.started":"2022-01-21T15:19:01.337152Z","shell.execute_reply":"2022-01-21T15:19:15.090498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby(['month','day','building_id','meter'])['meter_reading'].sum().sort_values(ascending = False).reset_index().head()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:19:15.092551Z","iopub.execute_input":"2022-01-21T15:19:15.092783Z","iopub.status.idle":"2022-01-21T15:19:25.175272Z","shell.execute_reply.started":"2022-01-21T15:19:15.092755Z","shell.execute_reply":"2022-01-21T15:19:25.174635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby(['month','day','building_id','meter'])['meter_reading'].sum().sort_values(ascending = False).reset_index().head(100)['building_id'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:19:25.176632Z","iopub.execute_input":"2022-01-21T15:19:25.176944Z","iopub.status.idle":"2022-01-21T15:19:34.898433Z","shell.execute_reply.started":"2022-01-21T15:19:25.176904Z","shell.execute_reply":"2022-01-21T15:19:34.897356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:19:34.899977Z","iopub.execute_input":"2022-01-21T15:19:34.900645Z","iopub.status.idle":"2022-01-21T15:19:35.080043Z","shell.execute_reply.started":"2022-01-21T15:19:34.900588Z","shell.execute_reply":"2022-01-21T15:19:35.079419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby(['building_id','site_id','meter'])['meter_reading'].sum().sort_values(ascending = False).reset_index().head(10)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:19:35.081197Z","iopub.execute_input":"2022-01-21T15:19:35.081603Z","iopub.status.idle":"2022-01-21T15:19:36.245406Z","shell.execute_reply.started":"2022-01-21T15:19:35.081572Z","shell.execute_reply":"2022-01-21T15:19:36.244756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.groupby(['building_id','site_id','meter'])['cons/sqft'].sum().sort_values(ascending = False).reset_index().head(10)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:19:36.246572Z","iopub.execute_input":"2022-01-21T15:19:36.246817Z","iopub.status.idle":"2022-01-21T15:19:37.662703Z","shell.execute_reply.started":"2022-01-21T15:19:36.246787Z","shell.execute_reply":"2022-01-21T15:19:37.662118Z"},"trusted":true},"execution_count":null,"outputs":[]}]}