{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport missingno as msno\nimport gc\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-05T22:13:04.880871Z","iopub.execute_input":"2022-01-05T22:13:04.881153Z","iopub.status.idle":"2022-01-05T22:13:04.891900Z","shell.execute_reply.started":"2022-01-05T22:13:04.881120Z","shell.execute_reply":"2022-01-05T22:13:04.890941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_PATH = \"../input/ashrae-energy-prediction/\"\nweather_train = pd.read_csv(DATA_PATH + 'weather_train.csv')\nweather_test = weather_df = pd.read_csv(DATA_PATH + 'weather_test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-01-05T22:13:05.519016Z","iopub.execute_input":"2022-01-05T22:13:05.519334Z","iopub.status.idle":"2022-01-05T22:13:05.960699Z","shell.execute_reply.started":"2022-01-05T22:13:05.519301Z","shell.execute_reply":"2022-01-05T22:13:05.959760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T22:13:06.058802Z","iopub.execute_input":"2022-01-05T22:13:06.059106Z","iopub.status.idle":"2022-01-05T22:13:06.079041Z","shell.execute_reply.started":"2022-01-05T22:13:06.059074Z","shell.execute_reply":"2022-01-05T22:13:06.077995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train[\"timestamp\"] = pd.to_datetime(weather_train[\"timestamp\"])\nweather_train[\"day\"]       = weather_train[\"timestamp\"].dt.day\nweather_train[\"hour\"]      = weather_train[\"timestamp\"].dt.hour\nweather_train[\"week\"]      = weather_train[\"timestamp\"].dt.weekday\nweather_train[\"month\"]     = weather_train[\"timestamp\"].dt.month\nweather_train              = weather_train.drop(\"timestamp\", axis = 1)\nweather_train","metadata":{"execution":{"iopub.status.busy":"2022-01-05T22:13:06.599739Z","iopub.execute_input":"2022-01-05T22:13:06.600206Z","iopub.status.idle":"2022-01-05T22:13:06.745032Z","shell.execute_reply.started":"2022-01-05T22:13:06.600169Z","shell.execute_reply":"2022-01-05T22:13:06.744053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T22:13:07.268503Z","iopub.execute_input":"2022-01-05T22:13:07.268794Z","iopub.status.idle":"2022-01-05T22:13:07.281614Z","shell.execute_reply.started":"2022-01-05T22:13:07.268762Z","shell.execute_reply":"2022-01-05T22:13:07.280519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Drop Column\ndrop_cols     = [\"precip_depth_1_hr\", \"sea_level_pressure\", \"wind_direction\", \"wind_speed\"]\nweather_train = weather_train.drop(drop_cols, axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-01-05T22:13:08.110659Z","iopub.execute_input":"2022-01-05T22:13:08.110944Z","iopub.status.idle":"2022-01-05T22:13:08.119729Z","shell.execute_reply.started":"2022-01-05T22:13:08.110911Z","shell.execute_reply":"2022-01-05T22:13:08.118608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,ax = plt.subplots(figsize=(12,5))\n\n#Plot\nfiltered_data = msno.nullity_filter(weather_train, filter='bottom', n=15, p=0.999)\nmsno.bar(filtered_data,fontsize=20)\n\nplt.xticks(fontsize=14, weight=\"bold\", alpha=0.75)\n\n#Title\nax.text(x=-0.5,y=1.2,s=\"Visualization of missing data on the training dataset.\",fontsize=30,weight=\"bold\",alpha=0.75)\nax.text(x=-0.5,y=1.15,s=\"The feature with the most missing data is floor_count where 82.6% of the data is missing.\",fontsize=22,alpha=0.75)\n#plt.savefig(\"missing_values_plot.png\")\n\ndel filtered_data\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T22:13:08.769520Z","iopub.execute_input":"2022-01-05T22:13:08.769803Z","iopub.status.idle":"2022-01-05T22:13:09.646199Z","shell.execute_reply.started":"2022-01-05T22:13:08.769771Z","shell.execute_reply":"2022-01-05T22:13:09.645161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#checking missing values for weather_train\ntotal           = weather_train.isnull().sum().sort_values(ascending=False)\npercent         = (weather_train.isnull().sum()/weather_train.isnull().count()*100).sort_values(ascending=False)\nmissing_wtrain  = pd.concat([total,percent],axis=1,keys=[\"Total\",\"Percent\"])\nmissing_wtrain","metadata":{"execution":{"iopub.status.busy":"2022-01-05T22:13:10.212614Z","iopub.execute_input":"2022-01-05T22:13:10.213664Z","iopub.status.idle":"2022-01-05T22:13:10.239239Z","shell.execute_reply.started":"2022-01-05T22:13:10.213618Z","shell.execute_reply":"2022-01-05T22:13:10.238568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Column in wtrain_df\nprint(f'\\nColumn types in wtrain_df::\\n{weather_train.dtypes.value_counts()}')","metadata":{"execution":{"iopub.status.busy":"2022-01-05T22:13:19.734886Z","iopub.execute_input":"2022-01-05T22:13:19.735432Z","iopub.status.idle":"2022-01-05T22:13:19.741854Z","shell.execute_reply.started":"2022-01-05T22:13:19.735397Z","shell.execute_reply":"2022-01-05T22:13:19.740942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T22:13:20.296016Z","iopub.execute_input":"2022-01-05T22:13:20.296338Z","iopub.status.idle":"2022-01-05T22:13:20.307442Z","shell.execute_reply.started":"2022-01-05T22:13:20.296297Z","shell.execute_reply":"2022-01-05T22:13:20.306236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(weather_train['cloud_coverage'].mean(),weather_train['dew_temperature'].mean(),weather_train['air_temperature'].mean(),)","metadata":{"execution":{"iopub.status.busy":"2022-01-05T22:13:20.958971Z","iopub.execute_input":"2022-01-05T22:13:20.959280Z","iopub.status.idle":"2022-01-05T22:13:20.968453Z","shell.execute_reply.started":"2022-01-05T22:13:20.959231Z","shell.execute_reply":"2022-01-05T22:13:20.967479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train['cloud_coverage'].fillna(weather_train['cloud_coverage'].mode()[0], inplace=True)\nweather_train['dew_temperature'].fillna(weather_train['dew_temperature'].mode()[0], inplace=True)\nweather_train['air_temperature'].fillna(weather_train['air_temperature'].mode()[0], inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-05T22:13:22.046905Z","iopub.execute_input":"2022-01-05T22:13:22.047466Z","iopub.status.idle":"2022-01-05T22:13:22.059999Z","shell.execute_reply.started":"2022-01-05T22:13:22.047415Z","shell.execute_reply":"2022-01-05T22:13:22.058907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Compare the differnece between after and before\nprint(weather_train['cloud_coverage'].mean(),weather_train['dew_temperature'].mean(),weather_train['air_temperature'].mean(),)","metadata":{"execution":{"iopub.status.busy":"2022-01-05T22:13:23.185202Z","iopub.execute_input":"2022-01-05T22:13:23.185521Z","iopub.status.idle":"2022-01-05T22:13:23.191587Z","shell.execute_reply.started":"2022-01-05T22:13:23.185489Z","shell.execute_reply":"2022-01-05T22:13:23.190661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}