{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g.L pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# importing libraries\n\n%matplotlib inline\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Reading the data and performing some basic statistics on given data frame using the methods like pandas describe() and info().\n\n#### To avoid memory issues or reducing the memory we'll supress the datatypes into 32-bit or 16-bit."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reading the data\n\n# Reading metadata\n# specify column dtypes to save memory (by default pandas reads some columns as floats)\ndtypes = {\n    \"site_id\": np.int8,\n    \"building_id\": np.int8,\n    \"primary_use\": np.object,\n    \"square_feet\": np.int8,\n    \"year_built\": np.float16,\n    \"floor_count\": np.float16,\n}\n\n# meta data information for metadata dataframe\nskiprows = None\nnrows = None\ncolumns = ['site_id', 'building_id', 'primary_use', 'square_feet', 'year_built', 'floor_count']\npath = \"/kaggle/input/ashrae-energy-prediction/building_metadata.csv\"\n\nmetadata_df = pd.read_csv(path, skiprows=skiprows, nrows=nrows, dtype=dtypes, usecols=columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reading training data\n\n# specify column dtypes to save memory (by default pandas reads some columns as floats)\ndtypes = {\n    \"building_id\": np.int8,\n    \"meter\": np.int8,\n    \"timestamp\": np.object,\n    \"meter_reading\": np.float16\n}\n\n# meta data information for metadata dataframe\nskiprows = None\nnrows = None\ncolumns = ['building_id', 'meter', 'timestamp', 'meter_reading']\npath = \"/kaggle/input/ashrae-energy-prediction/train.csv\"\n\n# using chunck size to avoid memory related issues\ntrain_df = pd.read_csv(path, skiprows=skiprows, nrows=nrows, dtype=dtypes, usecols=columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reading testing data\n\n# specify column dtypes to save memory (by default pandas reads some columns as floats)\ndtypes = {\n    \"row_id\": np.int8,\n    \"building_id\": np.int8,\n    \"meter\": np.int8,\n    \"timestamp\": np.object\n}\n\n# meta data information for metadata dataframe\nskiprows = None\nnrows = None\ncolumns = ['row_id', 'building_id', 'meter', 'timestamp']\npath = \"/kaggle/input/ashrae-energy-prediction/test.csv\"\n\n# using chunck size to avoid memory related issues\ntest_df = pd.read_csv(path, skiprows=skiprows, nrows=nrows, dtype=dtypes, usecols=columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reading weather training data\n\n# specify column dtypes to save memory (by default pandas reads some columns as floats)\ndtypes = {\n    \"site_id\": np.int8,\n    \"timestamp\": np.object,\n    \"air_temperature\": np.float16,\n    \"cloud_coverage\": np.float16,\n    \"dew_temperature\": np.float16,\n    \"precip_depth_1_hr\": np.float16,\n    \"sea_level_pressure\": np.float16,\n    \"wind_direction\": np.float16,\n    \"wind_speed\": np.float16\n}\n\n# meta data information for metadata dataframe\nskiprows = None\nnrows = None\ncolumns = ['site_id', 'timestamp', 'air_temperature', 'cloud_coverage',\n       'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',\n       'wind_direction', 'wind_speed']\npath = \"/kaggle/input/ashrae-energy-prediction/weather_train.csv\"\n\nweather_train_df = pd.read_csv(path, skiprows=skiprows, nrows=nrows, dtype=dtypes, usecols=columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reading weather testing data\n\n# specify column dtypes to save memory (by default pandas reads some columns as floats)\ndtypes = {\n    \"site_id\": np.int8,\n    \"timestamp\": np.object,\n    \"air_temperature\": np.float16,\n    \"cloud_coverage\": np.float16,\n    \"dew_temperature\": np.float16,\n    \"precip_depth_1_hr\": np.float16,\n    \"sea_level_pressure\": np.float16,\n    \"wind_direction\": np.float16,\n    \"wind_speed\": np.float16\n}\n\n# meta data information for metadata dataframe\nskiprows = None\nnrows = None\ncolumns = ['site_id', 'timestamp', 'air_temperature', 'cloud_coverage',\n       'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',\n       'wind_direction', 'wind_speed']\npath = \"/kaggle/input/ashrae-energy-prediction/weather_test.csv\"\n\nweather_test_df = pd.read_csv(path, skiprows=skiprows, nrows=nrows, dtype=dtypes, usecols=columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# describing about the data\n\nmetadata_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"metadata_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"metadata_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Basic EDA \n#### Performing some basic EDA which includes techniques like cleaning, feature engineering, transforming and standardising the data.\n#### Performing univariate analysis and bivariate analysis."},{"metadata":{"trusted":true},"cell_type":"code","source":"# checking the null values on metadata dataset\n\nmetadata_df.isnull().sum(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train_df.isnull().sum(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test_df.isnull().sum(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.isnull().sum(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.isnull().sum(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# percentage of missing values in weather training dataset\n\nround((weather_train_df.isnull().sum() * 100 / len(weather_train_df)), 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# percentage of missing values in weather testing dataset\n\nround((weather_test_df.isnull().sum() * 100 / len(weather_test_df)), 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Finding no.f unique values in weather training dataset\n\nprint(\"Finding no.f unique values in weather training dataset: {}\".format(len(pd.unique(weather_train_df['site_id']))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Finding no.f unique values in weather testing dataset\n\nprint(\"Finding no.f unique values in weather testing dataset: {}\".format(len(pd.unique(weather_test_df['site_id']))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Finding no.f unique values in training dataset \n\nprint(\"Finding no.f unique values in training dataset: {}\".format(len(pd.unique(train_df[\"building_id\"]))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Finding no.f unique values in testing dataset\n\nprint(\"Finding no.f unique values in testing dataset: {}\".format(len(pd.unique(test_df[\"building_id\"]))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# No.f records in weather training dataset\n\nprint(\"No.f records in weather training dataset: {}\".format(len(weather_train_df)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# No.f records in weather testing dataset\n\nprint(\"No.f records in weather testing dataset: {}\".format(len(weather_test_df)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# No.f records in trainging dataset\n\nprint(\"No.f records in training dataset: {}\".format(len(train_df)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# No.f records in testing dataset\n\nprint(\"No.f records in testing dataset: {}\".format(len(test_df)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Converting data types in the given dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train_df['timestamp'] = pd.to_datetime(weather_train_df['timestamp'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test_df['timestamp'] = pd.to_datetime(weather_test_df['timestamp'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[\"timestamp\"] = pd.to_datetime(train_df[\"timestamp\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df[\"timestamp\"] = pd.to_datetime(test_df[\"timestamp\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# removing unnecessary data\n\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}