{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Persiapan Data","metadata":{}},{"cell_type":"code","source":"import numpy as np #linear algebra \nimport pandas as pd #data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport warnings\n\n#input data files are available in read-only \"../input/\" directory\n#for example, running this (by clicking run or pressing Shift+Enter) will list all files unde the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \ntest_data = pd.read_csv(\"../input/ashrae-energy-prediction/test.csv\")\ntrain_data = pd.read_csv(\"../input/ashrae-energy-prediction/train.csv\")\nbuilding = pd.read_csv(\"../input/ashrae-energy-prediction/building_metadata.csv\")\nweathertest = pd.read_csv(\"../input/ashrae-energy-prediction/weather_test.csv\")\nweathertrain = pd.read_csv(\"../input/ashrae-energy-prediction/weather_train.csv\")\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:25:20.245201Z","iopub.execute_input":"2022-04-05T17:25:20.245911Z","iopub.status.idle":"2022-04-05T17:26:05.003654Z","shell.execute_reply.started":"2022-04-05T17:25:20.245867Z","shell.execute_reply":"2022-04-05T17:26:05.002672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#mengurangi penggunaan memori data, sumber: https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\ndef reduce_mem_usage(df, df_name):\n\n    start_mem = df.memory_usage().sum() / 1024**2\n    \n    for col in df.columns:\n\n        col_type = df[col].dtype\n\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n\n    print('Memory usage of {} is reduced by {:.2f} %. Usage dropped from {:.2f} MB to {:.2f} MB.'.format(df_name, (100 * (start_mem - end_mem) / start_mem), start_mem, end_mem))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:26:05.005602Z","iopub.execute_input":"2022-04-05T17:26:05.005844Z","iopub.status.idle":"2022-04-05T17:26:05.022078Z","shell.execute_reply.started":"2022-04-05T17:26:05.005815Z","shell.execute_reply":"2022-04-05T17:26:05.021203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = reduce_mem_usage(train_data, 'Train Data')\ntest_data = reduce_mem_usage(test_data, 'Test Data')\n\nbuilding = reduce_mem_usage(building, 'Building Data')\n\nweathertrain = reduce_mem_usage(weathertrain, 'Weather Train Data')\nweathertest = reduce_mem_usage(weathertest, 'Weather Test Data')","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:26:05.023456Z","iopub.execute_input":"2022-04-05T17:26:05.023999Z","iopub.status.idle":"2022-04-05T17:26:13.274899Z","shell.execute_reply.started":"2022-04-05T17:26:05.023966Z","shell.execute_reply":"2022-04-05T17:26:13.274189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#gabung table building_metadata, test, dan weather_test \ntest = building.merge(test_data, on ='building_id', how = 'left').merge(weathertest, on = ('site_id', 'timestamp'), how = 'left')\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:26:13.275892Z","iopub.execute_input":"2022-04-05T17:26:13.276094Z","iopub.status.idle":"2022-04-05T17:26:30.748715Z","shell.execute_reply.started":"2022-04-05T17:26:13.276068Z","shell.execute_reply":"2022-04-05T17:26:30.747839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#gabung tabel building_metadata, train, dan weather_train\ntrain = building.merge(train_data, on ='building_id', how ='left').merge(weathertrain, on = ('site_id', 'timestamp'), how ='left')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:26:30.751028Z","iopub.execute_input":"2022-04-05T17:26:30.751297Z","iopub.status.idle":"2022-04-05T17:26:39.012953Z","shell.execute_reply.started":"2022-04-05T17:26:30.751266Z","shell.execute_reply":"2022-04-05T17:26:39.01216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing Pada Tabel Train","metadata":{}},{"cell_type":"code","source":"#ubah tipe data pada kolom berikut\ntrain['site_id'] = train['site_id'].astype('category')\ntrain['building_id'] = train['building_id'].astype('category')\ntrain['meter'] = train['meter'].astype('category')\ntrain['timestamp'] = pd.to_datetime(train['timestamp'], format='%Y-%m-%d %H:%M:%S')","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:26:39.014113Z","iopub.execute_input":"2022-04-05T17:26:39.014393Z","iopub.status.idle":"2022-04-05T17:26:47.066739Z","shell.execute_reply.started":"2022-04-05T17:26:39.014364Z","shell.execute_reply":"2022-04-05T17:26:47.065936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:45:00.082845Z","iopub.execute_input":"2022-04-05T17:45:00.083294Z","iopub.status.idle":"2022-04-05T17:45:00.104748Z","shell.execute_reply.started":"2022-04-05T17:45:00.083257Z","shell.execute_reply":"2022-04-05T17:45:00.10416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#periksa info data pada tabel gabungan\ntrain.info()\ntrain.describe(include=\"all\")","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:26:47.067957Z","iopub.execute_input":"2022-04-05T17:26:47.0682Z","iopub.status.idle":"2022-04-05T17:27:23.519386Z","shell.execute_reply.started":"2022-04-05T17:26:47.068173Z","shell.execute_reply":"2022-04-05T17:27:23.51853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Pada kolom precip_depth_1_hr, terlihat bahwa nilai minimalnya -1. Padahal variabel precip_depth_1_hr merupakan ukuran panjang yang tidak bisa < 0. Nilai -1 akan diubah menjadi 0 dengan menganggap bahwa terjadi salah input.","metadata":{}},{"cell_type":"code","source":"train['precip_depth_1_hr']=train['precip_depth_1_hr'].replace(-1,0)\ntrain['precip_depth_1_hr'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:27:23.52077Z","iopub.execute_input":"2022-04-05T17:27:23.521106Z","iopub.status.idle":"2022-04-05T17:27:26.973077Z","shell.execute_reply.started":"2022-04-05T17:27:23.521079Z","shell.execute_reply":"2022-04-05T17:27:26.972292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Mencari Outlier","metadata":{}},{"cell_type":"markdown","source":"karena 'meter_reading' merupakan target variable, maka kita akan mencari apakah 'meter_reading' memiliki outlier atau tidak.","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 8))\nsns.distplot(train['meter_reading'], color='blue', ax = ax).set_title(\"Tabel Distribusi meter_reading\", fontsize=16)\nplt.xlabel('meter_reading', fontsize=12)\nplt.ylabel('Density', fontsize=12)\nplt.show()\n\nsns.boxplot(x=train['meter_reading'])","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:27:26.974372Z","iopub.execute_input":"2022-04-05T17:27:26.974589Z","iopub.status.idle":"2022-04-05T17:28:44.895722Z","shell.execute_reply.started":"2022-04-05T17:27:26.974562Z","shell.execute_reply":"2022-04-05T17:28:44.894743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"data akan ditransformasi dengan transformasi logaritma untuk mengurangi efek outlier","metadata":{}},{"cell_type":"code","source":"train['meter_reading']=np.log1p(train['meter_reading'])\n\nfig, ax = plt.subplots(figsize=(8, 8))\nsns.distplot(train['meter_reading'], color='blue', ax = ax).set_title(\"Tabel Distribusi meter_reading\", fontsize=16)\nplt.xlabel('meter_reading', fontsize=12)\nplt.ylabel('Density', fontsize=12)\nplt.show()\n\nsns.boxplot(x=train['meter_reading'])","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:28:44.897302Z","iopub.execute_input":"2022-04-05T17:28:44.897524Z","iopub.status.idle":"2022-04-05T17:30:00.255247Z","shell.execute_reply.started":"2022-04-05T17:28:44.897497Z","shell.execute_reply":"2022-04-05T17:30:00.253603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"setelah transformasi, efek outlier berkurang","metadata":{}},{"cell_type":"markdown","source":"## Missing Values","metadata":{}},{"cell_type":"code","source":"train.isnull().sum() #jumlah missing values pada setiap kolom","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:30:00.256381Z","iopub.execute_input":"2022-04-05T17:30:00.256612Z","iopub.status.idle":"2022-04-05T17:30:01.434601Z","shell.execute_reply.started":"2022-04-05T17:30:00.256583Z","shell.execute_reply":"2022-04-05T17:30:01.433877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#persentase missing values terhadap jumlah data\nmissing_values = pd.DataFrame(train.isnull().sum() * 100 / len(train))\nmissing_values.columns = [\"Missing Values\"]\n\nmissing_values = missing_values[missing_values[\"Missing Values\"] != 0]\nmissing_values.sort_values(by = \"Missing Values\", axis = 0, ascending = False, inplace = True)\n\nmissing_values","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:30:01.43572Z","iopub.execute_input":"2022-04-05T17:30:01.436011Z","iopub.status.idle":"2022-04-05T17:30:02.617022Z","shell.execute_reply.started":"2022-04-05T17:30:01.435981Z","shell.execute_reply":"2022-04-05T17:30:02.616252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,8))\nsns.barplot(x = missing_values[\"Missing Values\"], y = missing_values.index).set_title(\"Persentase Missing Values\", fontsize=16)\nplt.xlabel(\"Persen(%)\",fontsize=12)\nplt.ylabel(\"Variabel\",fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:30:02.618164Z","iopub.execute_input":"2022-04-05T17:30:02.618449Z","iopub.status.idle":"2022-04-05T17:30:02.86447Z","shell.execute_reply.started":"2022-04-05T17:30:02.61842Z","shell.execute_reply":"2022-04-05T17:30:02.863805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dari tabel dan visualisasi tersebut, dapat dilihat bahwa kolom 'floor_count' dan 'year_built' memiliki persentase missing values > 50% dan variabel-variabel tersebut kurang penting sehingga akan di-delete.","metadata":{}},{"cell_type":"code","source":"#drop kolom dengan persentase missing values > 50%\ntrain.drop('floor_count', axis=1, inplace=True)\ntrain.drop('year_built', axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:30:02.866844Z","iopub.execute_input":"2022-04-05T17:30:02.867088Z","iopub.status.idle":"2022-04-05T17:30:04.575078Z","shell.execute_reply.started":"2022-04-05T17:30:02.867058Z","shell.execute_reply":"2022-04-05T17:30:04.574194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#tabel histogram untuk periksa distribusi variabel\ntrain.hist(bins=50, figsize=(24,15))\nplt.suptitle('Histogram', fontsize=16)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:30:04.576771Z","iopub.execute_input":"2022-04-05T17:30:04.576997Z","iopub.status.idle":"2022-04-05T17:30:25.84818Z","shell.execute_reply.started":"2022-04-05T17:30:04.576971Z","shell.execute_reply":"2022-04-05T17:30:25.847573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Untuk variabel dengan distribusi yang cukup skewed, maka akan diimputasi dengan median, Sedangkan untuk variabel dengan distribusi yang tidak skewed/mendekati normal, maka akan diimputasi dengan mean.\n\nKarena semua variabel cukup skewed, maka akan diimputasi dengan median","metadata":{}},{"cell_type":"code","source":"#imputasi dengan median\ntrain['cloud_coverage'].fillna(train['cloud_coverage'].median(), inplace=True)\ntrain['dew_temperature'].fillna(train['dew_temperature'].median(), inplace=True)\ntrain['precip_depth_1_hr'].fillna(train['precip_depth_1_hr'].median(), inplace=True)\ntrain['wind_direction'].fillna(train['wind_direction'].median(), inplace=True)\ntrain['wind_speed'].fillna(train['wind_speed'].median(), inplace=True)\ntrain['air_temperature'].fillna(train['air_temperature'].median(), inplace=True)\ntrain['sea_level_pressure'].fillna(train['sea_level_pressure'].median(), inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:30:25.849419Z","iopub.execute_input":"2022-04-05T17:30:25.849801Z","iopub.status.idle":"2022-04-05T17:30:29.22738Z","shell.execute_reply.started":"2022-04-05T17:30:25.849772Z","shell.execute_reply":"2022-04-05T17:30:29.226696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#data setelah preprocessing\ntrain ","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:44:18.97749Z","iopub.execute_input":"2022-04-05T17:44:18.978268Z","iopub.status.idle":"2022-04-05T17:44:19.006875Z","shell.execute_reply.started":"2022-04-05T17:44:18.978235Z","shell.execute_reply":"2022-04-05T17:44:19.006307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:44:23.117275Z","iopub.execute_input":"2022-04-05T17:44:23.117766Z","iopub.status.idle":"2022-04-05T17:44:24.040859Z","shell.execute_reply.started":"2022-04-05T17:44:23.117733Z","shell.execute_reply":"2022-04-05T17:44:24.040054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:45:21.141859Z","iopub.execute_input":"2022-04-05T17:45:21.142386Z","iopub.status.idle":"2022-04-05T17:45:22.039232Z","shell.execute_reply.started":"2022-04-05T17:45:21.142347Z","shell.execute_reply":"2022-04-05T17:45:22.038383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Analisis Data","metadata":{}},{"cell_type":"code","source":"#periksa statistika sederhana setelah preprocessing\ntrain.describe(include=\"all\")","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:30:30.146627Z","iopub.execute_input":"2022-04-05T17:30:30.146847Z","iopub.status.idle":"2022-04-05T17:30:57.307554Z","shell.execute_reply.started":"2022-04-05T17:30:30.146818Z","shell.execute_reply":"2022-04-05T17:30:57.306707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.hist(bins=50, figsize=(24,15))\nplt.suptitle('Histogram Setelah Preprocessing', fontsize=16)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:30:57.308731Z","iopub.execute_input":"2022-04-05T17:30:57.309363Z","iopub.status.idle":"2022-04-05T17:31:20.271853Z","shell.execute_reply.started":"2022-04-05T17:30:57.309327Z","shell.execute_reply":"2022-04-05T17:31:20.271037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,8))\nsns.countplot(x = 'site_id', data = train).set_title(\"Jumlah bangunan per site id\", fontsize=12)\nplt.xlabel('Site id')\nplt.ylabel('Jumlah Bangunan')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T18:40:21.980622Z","iopub.execute_input":"2022-04-05T18:40:21.980881Z","iopub.status.idle":"2022-04-05T18:40:22.771906Z","shell.execute_reply.started":"2022-04-05T18:40:21.980855Z","shell.execute_reply":"2022-04-05T18:40:22.771045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,8))\nsns.countplot(x = 'primary_use', data = train).set_title(\"Frekuensi kategori primary_use\", fontsize=12)\nplt.xticks(rotation=90)\nplt.xlabel('Primary_use')\nplt.ylabel('Count')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T17:58:39.621157Z","iopub.execute_input":"2022-04-05T17:58:39.62158Z","iopub.status.idle":"2022-04-05T17:58:40.480076Z","shell.execute_reply.started":"2022-04-05T17:58:39.621535Z","shell.execute_reply":"2022-04-05T17:58:40.479399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"building_groups = train.groupby(['building_id']).meter_reading.mean().reset_index()\n\nplt.figure(figsize=(14,8))\nsns.lineplot(x = building_groups['building_id'], y = building_groups.meter_reading).set_title(\"Rata-Rata Meter Reading berdasarkan building id\", fontsize=16)\nplt.xlabel('Building id', fontsize=12)\nplt.ylabel('Rata-Rata Meter Reading', fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T18:23:40.142757Z","iopub.execute_input":"2022-04-05T18:23:40.14366Z","iopub.status.idle":"2022-04-05T18:23:40.72131Z","shell.execute_reply.started":"2022-04-05T18:23:40.143599Z","shell.execute_reply":"2022-04-05T18:23:40.720464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"building_groups = train.groupby(['timestamp']).meter_reading.mean().reset_index()\n\nplt.figure(figsize=(14,8))\nsns.lineplot(x = building_groups['timestamp'], y = building_groups.meter_reading).set_title(\"Rata-Rata Meter Reading berdasarkan timestamp\", fontsize=16)\nplt.xlabel('Time Stamp', fontsize=12)\nplt.ylabel('Rata-Rata Meter Reading', fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T18:18:43.374694Z","iopub.execute_input":"2022-04-05T18:18:43.374979Z","iopub.status.idle":"2022-04-05T18:18:44.528052Z","shell.execute_reply.started":"2022-04-05T18:18:43.374949Z","shell.execute_reply":"2022-04-05T18:18:44.527204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#korelasi antar variabel\ncorrelation_matrix = train.corr()\nplt.figure(figsize=(18,18))\nsns.heatmap(correlation_matrix, annot = True, vmin = -1, vmax = 1, cmap=\"Blues\").set_title('Heatmap Korelasi')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-05T18:11:44.70365Z","iopub.execute_input":"2022-04-05T18:11:44.703938Z","iopub.status.idle":"2022-04-05T18:11:51.414335Z","shell.execute_reply.started":"2022-04-05T18:11:44.70391Z","shell.execute_reply":"2022-04-05T18:11:51.413428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.to_csv('EDA_Kelompok_F.csv',index=False)","metadata":{},"execution_count":null,"outputs":[]}]}