{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.offline as py\nimport plotly.graph_objs as go","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/ashrae-energy-prediction/train.csv\")\ntest = pd.read_csv(\"../input/ashrae-energy-prediction/test.csv\")\nweather_train = pd.read_csv(\"../input/ashrae-energy-prediction/weather_train.csv\")\nweather_test = pd.read_csv(\"../input/ashrae-energy-prediction/weather_test.csv\")\nbuilding_meta = pd.read_csv(\"../input/ashrae-energy-prediction/building_metadata.csv\")\nsample_sub = pd.read_csv(\"../input/ashrae-energy-prediction/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"building_meta.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.merge(building_meta, left_on = \"building_id\", right_on = \"building_id\", how = \"left\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.merge(weather_train, left_on = [\"site_id\", \"timestamp\"], right_on = [\"site_id\", \"timestamp\"], how = \"left\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = test.merge(building_meta, left_on = \"building_id\", right_on = \"building_id\", how = \"left\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = test.merge(weather_test, left_on = [\"site_id\", \"timestamp\"], right_on = [\"site_id\", \"timestamp\"], how = \"left\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print (\"Train: \",train.shape[0],\" and \",train.shape[1],\"features\")\nprint (\"Test:  \",test.shape[0],\" and \",test.shape[1],\"features\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndef reduce_mem_usage(df):\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df\ntrain = reduce_mem_usage(train)\ntest = reduce_mem_usage(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.isnull(train).any()\n\n#most of the columns have null values ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set_style(\"whitegrid\")\nmissing = train.isnull().sum()\nmissing = missing[missing > 0]\nmissing.sort_values(inplace=True)\nmissing.plot.bar()\n\n#floor_count has most missing values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Separate categorical and numerical columns\ncat_column = train.dtypes[train.dtypes == 'object']\nnum_column = train.dtypes[train.dtypes != 'object']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in list(cat_column.index):\n    print(f\"--------------------{col.title()}-------------------------\")\n    total= train[col].value_counts()\n    percent = total / train.shape[0]\n    df = pd.concat([total,percent],keys = ['total','percent'],axis = 1)\n    print(df)\n    print('\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.style.use('seaborn-whitegrid')\nplt.figure(figsize=(20,5)) \nsns.countplot(y=\"meter\", data=train);\n \n#0: electricity, 1: chilledwater, 2: steam, hotwater: 3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.style.use('seaborn-colorblind')\nplt.figure(figsize=(20,8)) \nsns.countplot(y=\"primary_use\", data=train);\n\n#major of the building are education, Office, Public assembly, Residental and Public services","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize = (12,10))\n\nsns.heatmap(train[list(num_column.index)].corr(),annot = True,square = True);\n\n#no features are highly corelated except few of them like site_id to building_id, air_temprature to dew_temperature","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.style.use('seaborn-colorblind')\nplt.figure(figsize=(20,40)) \nsns.countplot(y=\"year_built\", data=train);\n\n#the data contains building from 1900 to 2019\n# Highest number of building were built in 1976","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count = building_meta[\"floor_count\"].value_counts()\ntrace = go.Bar(\n    x=count.index,\n    y=count.values,\n    marker=dict(\n        color=\"green\",\n    ),\n)\nlayout = go.Layout(\n    title=go.layout.Title(\n        text=\"Floor Count\",\n        x=0.5\n    ),\n    font=dict(size=14),\n    width=1000,\n    height=500,\n)\ndata = [trace]\nfig = go.Figure(data=data, layout=layout)\npy.iplot(fig, filename=\"meter\")\n\n#There are many buildings 1 floor and as floor counts increases, the number of buildings decreases. \n#Only 1 building exists in our data with 26 floors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Work in Progress !!! \n#Please upvote if you like it !!!","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}