{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Introduction\nHello  \nThis is my first attempt at exploratory data analysis. In this notebook, I would like to try to describe the data of the current competition, to find oddities and patterns in the data.  \nI will be glad to advice and comments!)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom sklearn.preprocessing import LabelEncoder\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# building_metadata.csv exploration"},{"metadata":{},"cell_type":"markdown","source":"Load data and describe it a bit"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"building_df = pd.read_csv(\"../input/ashrae-energy-prediction/building_metadata.csv\")\nprint(building_df.shape)\nbuilding_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"building_df.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Check data types of our columns. Many columns have a larger data type than they should:"},{"metadata":{"trusted":true},"cell_type":"code","source":"building_df.dtypes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Missed Data Percentage"},{"metadata":{"trusted":true},"cell_type":"code","source":"building_df.isna().sum() / building_df.shape[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We set a smaller data type for columns that do not have missing values (we will work with the remaining columns later)"},{"metadata":{"trusted":true},"cell_type":"code","source":"building_df['site_id'] = building_df['site_id'].astype(np.uint8)\nbuilding_df['building_id'] = building_df['building_id'].astype(np.uint16)\nbuilding_df['square_feet'] = building_df['square_feet'].astype(np.uint32)\nbuilding_df.dtypes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's look at the distribution of data on various features. For the site_id and primary_use, you can use the bar plot"},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_columns = ['site_id', 'primary_use']\nplt.figure(figsize=(15, 5))\nfor ind, col in enumerate(cat_columns):\n    plt.subplot(1, len(cat_columns), ind+1)\n    plt.title(col)\n    building_df[col].value_counts(sort=False).plot(kind='bar')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"For year_build we can use simple plot, and see how many building was build in each year"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,5))\ny = building_df['year_built'].value_counts(sort=False)\nplt.title('year_built')\nplt.xticks(rotation=45)\ny.index = pd.to_datetime(y.index.astype(int), format='%Y')\ny = y.sort_index()\nplt.grid()\nplt.plot(y)\ndel y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's convert prinary_use column to the number:"},{"metadata":{"trusted":true},"cell_type":"code","source":"le = LabelEncoder()\nbuilding_df[\"primary_use_enc\"] = le.fit_transform(building_df[\"primary_use\"]).astype(np.uint8)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"And create correlation matrix for our data. As we can see squeree_feet and floor_count is correlate well. This fact may be useful in recovering missing values for the number of floors."},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(6,6))\nsns.heatmap(building_df.corr(), square=True, annot=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's see the dependence of the number of floors on the logarithm of the area"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(5, 5))\nplt.xlabel('square_feet')\nplt.ylabel('floor_count')\nX = building_df[pd.notnull(building_df['floor_count'])]\nplt.scatter(np.log1p(X['square_feet']), X['floor_count'])\ndel X","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# weather_train.csv exploration"},{"metadata":{},"cell_type":"markdown","source":"Load data and describe it a bit"},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train = pd.read_csv(\"../input/ashrae-energy-prediction/weather_train.csv\")\nprint(weather_train.shape)\nweather_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train.dtypes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Missed data percentage"},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train.isna().sum() / weather_train.shape[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Change dtypes of the columns"},{"metadata":{"trusted":true},"cell_type":"code","source":"float_cols = ['air_temperature', 'cloud_coverage', 'dew_temperature',\n        'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train['site_id'] = weather_train['site_id'].astype(np.uint8)\nweather_train['timestamp'] = pd.to_datetime(weather_train['timestamp'])\nfor i in float_cols:\n    weather_train[i] = weather_train[i].astype(np.float32)\nweather_train.dtypes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Check correlations between features "},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(8, 8))\nsns.heatmap(weather_train.corr(), square=True, annot=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp = weather_train[weather_train['site_id'] == 0]\nplt.figure(figsize=(20, 15))\nfor ind, col in enumerate(float_cols):\n    plt.subplot(4, 2, ind + 1)\n    plt.xticks(rotation=30)\n    plt.grid()\n    plt.title(col)\n    plt.plot(tmp['timestamp'], tmp[col])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### more later..."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}