{"cells":[{"metadata":{"_uuid":"5fea04a7-f194-4dc0-bf25-0696f93248ea","_cell_guid":"fd906c05-0ef3-4c6c-b610-9db95d5f5937","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n!pip install plotly==4.4.1\nimport plotly.express as px\nimport plotly.figure_factory as ff\nimport plotly.graph_objects as go\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Bu kisim kaggle optimizasyonu icin yapildi.\n>  Memory reduction etc."},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"metadata_dtype = {'site_id':\"uint8\",'building_id':'uint16','square_feet':'float32','year_built':'float32','floor_count':\"float16\"}\nmetadata = pd.read_csv(\"../input/ashrae-energy-prediction/building_metadata.csv\",dtype=metadata_dtype)\nmetadata.info(memory_usage='deep')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_dtype = {\"site_id\":\"uint8\"}\nweather_train = pd.read_csv(\"../input/ashrae-energy-prediction/weather_train.csv\",parse_dates=['timestamp'],dtype=weather_dtype)\nweather_test = pd.read_csv(\"../input/ashrae-energy-prediction/weather_test.csv\",parse_dates=['timestamp'],dtype=weather_dtype)\nprint (weather_train.info(memory_usage='deep'))\nprint (\"-------------------------------------\")\nprint (weather_test.info(memory_usage='deep'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### 21 milyon satir icerisinden sadece 1 milyonunu aliyorum"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dtype = {'meter':\"uint8\",'building_id':'uint16','meter_reading':\"float32\"}\ntrain = pd.read_csv(\"../input/ashrae-energy-prediction/train.csv\",parse_dates=['timestamp'],dtype=train_dtype)\ntrain = train.sample(n=100000, random_state=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"43a1567a-73e4-4a9c-a2c6-4770c4fdb50e","_cell_guid":"81214498-e1df-4134-bb02-16a222c36dac","trusted":true},"cell_type":"code","source":"# Burda herseyi birlestirme zamani\n\nnew = pd.merge(left= train,right=metadata,how='left' ,left_on ='building_id', right_on = 'building_id')\ndata = pd.merge(left = new, right=weather_train, how='left', left_on=['site_id','timestamp'],right_on= ['site_id','timestamp'])\ndata = reduce_mem_usage(data)\n# Burda building metadata ile train dosyasini birlestirdik\n\n# Memory kaybetmemek icin gereksizleri siliyoruz.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Memory kullanimini optimize etmek icin gereksiz frameleri siliyoruz."},{"metadata":{"trusted":true},"cell_type":"code","source":"del train\ndel weather_train\ndel weather_test\ndel metadata","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Descriptive Analysis"},{"metadata":{"trusted":true},"cell_type":"code","source":"data.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Outlierlari cikariyoruz"},{"metadata":{"trusted":true},"cell_type":"code","source":"q1 = data['meter_reading'].quantile(.25)\nprint('q1 is {}'.format(q1))\nq3 = data['meter_reading'].quantile(.75)\nprint('q3 is {}'.format(q3))\nIQR_main = q3- q1\n#mask = data['meter_reading'].between(0, 1.5 * IQR_main, inclusive=True)\n#iqr = data.loc[mask, 'meter_reading']\n\ndata.drop(data[(data['meter_reading'] < 0) | (data['meter_reading'] > (1.5 * IQR_main)) ].index , inplace=True)\nprint(data['meter_reading'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Burda sayac bilgimizi gormek icin degerleri float sekline getiriyoruz"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(data.info())\n#Scientific notation yapmasin diye float yapiyoruz\ndata['meter_reading'].describe().apply(lambda x: format(x, 'f'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Degiskeni incelemeden once biraz uzerinde oynama yapicaz.\n# NaN olanlari elememiz gerekiyor\n\nprint(np.isnan(data['meter_reading']).value_counts())\nprint(np.isinf(data['meter_reading']).value_counts())\n\n# NaN var ancak inf yok\n\n#data['meter_reading'] = np.nan_to_num(data['meter_reading'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c4c532f7-5b7b-47ca-9d48-fd23a0e510c8","_cell_guid":"f95f836f-aba5-4b35-869c-13b0700ae2d8","trusted":true},"cell_type":"code","source":"# Null satirlar var mi ? Var meter_reading'i degistirdik. Degismeyenler NULL olarak geldi. Onlari eleyecegiz.\nprint(data.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Wind Speed ve Direction uzerinden bi ilerleyelim"},{"metadata":{"trusted":true},"cell_type":"code","source":"data.dropna(subset =['wind_speed','wind_direction'],inplace = True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Datamizda kirlilik var ruzgar verileri icerisindeki NaN degerleri siliyoruz"},{"metadata":{"trusted":true},"cell_type":"code","source":"def funct(a):\n    if a >= 0 and a<= 1:\n        return '0-1'\n    elif a > 1 and a<= 2:\n        return '1-2'\n    elif a > 2 and a<= 3:\n        return '2-3'\n    elif a > 3 and a<= 4:\n        return '3-4'\n    elif a > 4 and a<= 5:\n        return '4-5'\n    elif a > 5 and a<= 6:\n        return '5-6'\n    else :\n        return '6+'\ndata['wind_speed'] = data['wind_speed'].apply(funct)\ndata['wind_speed'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Burda wind icin gerekli analizi yapmalik hale getirdik."},{"metadata":{"trusted":true},"cell_type":"code","source":"wind_data = data.groupby(['wind_speed','wind_direction']).size()\nwind_data = wind_data.reset_index()\nwind_data.columns = ['wind_speed', 'wind_direction','frequency']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Duplike satirimiz yok"},{"metadata":{"_uuid":"392492bd-fd65-4a52-b5f4-5141880c35c7","_cell_guid":"3245cfdf-c94a-4f14-8132-5a6e17cf1fc6","trusted":true},"cell_type":"code","source":"# Tekrar eden satirlar var mi ?\nprint(data.shape)\nduplicate_rows_df = data[data.duplicated()]\nprint('number of duplicate rows: ', duplicate_rows_df.shape)\n# Hic tekrar eden satirimiz yok !","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Tum bu degisikliklerden sonra index temizligi yapiyoruz.\n> Tarih degiskenini yeni kolon halinda datamiza ekliyoruz.\n> Daha rahat olsun diye meter degiskeninin numerikten string haline ceviriyoruz."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Birlestirmeden sonra tarihlerde ufak degisiklikler\n\nfor df in [data]:\n    df['Month'] = df['timestamp'].dt.month.astype(\"uint8\")\n    df['DayOfMonth'] = df['timestamp'].dt.day.astype(\"uint8\")\n    df['DayOfWeek'] = df['timestamp'].dt.weekday.astype(\"uint8\")\n    df['Hour'] = df['timestamp'].dt.hour.astype(\"uint8\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Burdaki meter degiskeni kategorik bir degisken ondan descriptive stat anlamsiz.\n\ndata['meter'].replace({0:\"Electricity\",1:\"ChilledWater\",2:\"Steam\",3:\"HotWater\"},inplace=True)\ndata['DayOfWeek'].replace({0:'Sunday',1:'Monday',2:'Tuesday',3:'Wednesday',4:'Thursday',5:'Friday',6:'Saturday'},inplace=True)\ndata.describe(include= 'all')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EDA IN DATA"},{"metadata":{},"cell_type":"markdown","source":"#### Dagilimina bakmak icin LOG donusumu yapiyoruz."},{"metadata":{"trusted":true},"cell_type":"code","source":"#small_data = data\n#small_data['meter_reading'] = np.log1p(data['meter_reading'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"fig = px.histogram(data, x=\"meter_reading\",nbins = 20)\nfig.update_layout(\n    title='<b>Distribution of Meter Reading</b><br><span> Without Outlier and Log Transform<br>',font=dict(\n        family=\"Courier New, monospace\",\n        size=18,\n        color=\"#000\"\n    ))\nfig.show(renderer=\"kaggle\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Degerlerimiz arasindaki korelasyon ne durumda onu inceleyelim "},{"metadata":{"_uuid":"78537512-a6ba-4235-ba71-21dbb01a0ca5","_cell_guid":"a49e0f71-67a5-4534-8bd6-0a57ff3877e3","trusted":true},"cell_type":"code","source":"c = data.corr()\n\nmask = np.zeros_like(c, dtype=np.bool)\nmask[np.triu_indices_from(mask)] = True\n\nf, ax = plt.subplots(figsize=(11, 9))\n\ncmap = sns.diverging_palette(220, 10, as_cmap=True)\n\nsns.heatmap(c, mask=mask, cmap=cmap, vmax=.3, center=0,\n            square=True, linewidths=.5, cbar_kws={\"shrink\": .5})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x1 = np.log1p(data[data['meter']=='Electricity']['meter_reading'])\nx2 = np.log1p(data[data['meter']=='ChilledWater']['meter_reading'])\nx3 = np.log1p(data[data['meter']=='Steam']['meter_reading'])\nx4 = np.log1p(data[data['meter']=='HotWater']['meter_reading'])\n              \nhist_data = [x1, x2, x3, x4]\ncolors = ['#835AF1', '#7FA6EE', '#B8F7D4','#F66095']              \ngroup_data = ['Electricity', 'ChilledWater', 'Steam', 'HotWater']\n              \nfig = ff.create_distplot(hist_data, group_data,colors=colors, bin_size = .1)\nfig.update_layout(\n    title='Distribution of Meter Reading by Meter Type')\nfig.show(renderer='kaggle')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f3250ed2-b59f-4c1e-8720-b71b06f0b04c","_cell_guid":"3714dd6b-8ea3-4fe8-8bbe-7254f9ec9b7e","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"'''y = data['wind_direction']\ny = y/57.324\nx = data['wind_speed']\nfig = plt.figure(figsize = (18,18))\nax = fig.add_subplot(111, projection='polar')\nc = ax.scatter(y,x, s= x * 10,c = data['site_id'],cmap='twilight',alpha=0.5)'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"fig = px.bar_polar(wind_data, r =wind_data['frequency'], theta=wind_data[\"wind_direction\"],\n                   color=wind_data['wind_speed'], template=\"plotly_dark\",\n                   color_discrete_sequence= px.colors.sequential.Plasma[-2::-1])\nfig.update_layout(title = 'Wind Polar Bar')                  \nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Meter_reading degerimizin primary use'a gore dagilimini bulalim bakalim"},{"metadata":{"trusted":true},"cell_type":"code","source":"meter_data = data.loc[:,['meter_reading','primary_use','meter','site_id']]\nmeter_dist = meter_data.loc[:,['meter_reading','primary_use','meter']]\nmeter_dist['index'] = meter_dist.reset_index().index\nmeter_dist.reset_index()\n\n# Burda yapmaya calistigim birazdan daha net belli olacak","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.violin(meter_dist, y=\"meter_reading\", color = 'primary_use')\nfig.update_layout(\n    title='Distribution of Meter Reading by Primary Use')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"import plotly.graph_objects as go\nfig = go.Figure()\nfig.add_trace(go.Violin(x=meter_dist['primary_use'][ meter_dist['meter'] == 'Electricity' ],\n                        y=df['meter_reading'][ df['meter'] == 'Electricity' ],\n                        legendgroup='Yes', scalegroup='No', name='Electricity',\n                        side='positive',\n                        line_color='blue')\n             )\nfig.add_trace(go.Violin(x=meter_dist['primary_use'][ meter_dist['meter'] == 'Steam' ],\n                        y=df['meter_reading'][ df['meter'] == 'Steam' ],\n                        legendgroup='No', scalegroup='No', name='Steam',\n                        side='negative',\n                        line_color='orange')\n             )\nfig.add_trace(go.Violin(x=meter_dist['primary_use'][ meter_dist['meter'] == 'ChilledWater' ],\n                        y=df['meter_reading'][ df['meter'] == 'ChilledWater' ],\n                        legendgroup='No', scalegroup='No', name='ChilledWater',\n                        side='positive',\n                        line_color='green')\n             )\nfig.add_trace(go.Violin(x=meter_dist['primary_use'][ meter_dist['meter'] == 'HotWater' ],\n                        y=df['meter_reading'][ df['meter'] == 'HotWater' ],\n                        legendgroup='No', scalegroup='No', name='HotWater',\n                        side='negative',\n                        line_color='red')\n             )\nfig.update_layout(\n    title='Distribution of Meter Reading by Primary Use and Meter')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Bu sekilde de bir bakalim"},{"metadata":{"_uuid":"ff3fc2c6-f5e3-43a7-b178-feec37e918e0","_cell_guid":"12931a1f-deb9-4391-9e37-8a6c25c1ec6c","trusted":true},"cell_type":"code","source":"fig = px.box(data, x=\"site_id\", y=\"air_temperature\", color = 'site_id')\nfig.update_layout(\n    title='Distribution of Air Temperature by Site')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<b>Haftanin zamanagore dagilimi"},{"metadata":{"trusted":true},"cell_type":"code","source":"time_data = data.loc[:,['timestamp', 'meter_reading','meter','site_id','DayOfMonth','Month','DayOfWeek','Hour','primary_use']]\ntime_data['meter_reading'] = np.log1p(time_data['meter_reading'])\ntime_data.reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.box(time_data, x=\"primary_use\", y=\"meter_reading\")\nfig.update_layout(\n    title='Distribution of Meter Reading by Primary Use')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.box(time_data, x=\"DayOfWeek\", y=\"meter_reading\")\nfig.update_layout(\n    title='Distribution of Meter Reading by Day')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Saatlik bu sekilde bir de total yillik bakalim"},{"metadata":{"trusted":true},"cell_type":"code","source":"time_temp = time_data.groupby(['Hour']).median().reset_index()\nfig = px.line(time_temp, x=\"Hour\", y=\"meter_reading\", title='<b>Meter Reading by Hour')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"time_temp = time_data.groupby(['Hour','primary_use']).median().reset_index()\nfig = px.line(time_temp, x=\"Hour\", y=\"meter_reading\", title='<b>Meter Reading by Hour with Primary Use', facet_col='primary_use', facet_col_wrap=4)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"time_temp = time_data.groupby(['DayOfWeek','primary_use']).median().reset_index()\n\nfig = px.box(time_data, x=\"meter_reading\", y=\"DayOfWeek\",facet_col='primary_use', facet_col_wrap=4)\nfig.update_layout(\n    title='Distribution of Meter Reading by Primary Use')\nfig.update_traces(orientation='h')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"time_temp = time_data.groupby(['timestamp']).median().reset_index()\n\nfig = px.line(time_temp, x=\"timestamp\", y=\"meter_reading\", title='<b>Meter Reading by Hour')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}