{"cells":[{"metadata":{},"cell_type":"markdown","source":"# About this kernel\n\n**The purpose of this notbook is do a little help :)**\n\nData Types used in these data files are not optimized. So you can reduce **50% memory** usages by changing datatypes according to data range.\n\n"},{"metadata":{},"cell_type":"markdown","source":"## Load Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_PATH = '../input/ashrae-energy-prediction/'\nbuilding_df = pd.read_csv(DATA_PATH + 'building_metadata.csv')\ntrain_df = pd.read_csv(DATA_PATH + 'train.csv')\ntest_df = pd.read_csv(DATA_PATH + 'test.csv')\nweather_train_df = pd.read_csv(DATA_PATH + 'weather_train.csv')\nweather_test_df = pd.read_csv(DATA_PATH + 'weather_test.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Utility Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def ds_size(dataset):\n    return (dataset.memory_usage().sum()) / (1024 ** 2)\n\ndef ds_statistics(df_names,df_lists) :\n    datasets_df = pd.DataFrame(columns=['Dataset','Number of Rows','Number of Columns','Size (MB)'])\n    rows = [df.shape[0] for df in df_lists]\n    columns = [df.shape[1] for df in df_lists]\n    size_MB = [ds_size(df) for df in df_lists]\n\n    datasets_df['Dataset'] = df_names\n    datasets_df['Number of Rows'] = rows\n    datasets_df['Number of Columns'] = columns\n    datasets_df['Size (MB)'] = size_MB\n    datasets_df = datasets_df.set_index('Dataset')\n    datasets_df = datasets_df.sort_values('Size (MB)')\n    return datasets_df\n\ndef ds_optimization(dataframe,categorical = []):\n    df = dataframe.copy()\n    int_types = {np.int8 : (np.iinfo(np.int8).min,np.iinfo(np.int8).max),\n                 np.int16: (np.iinfo(np.int16).min,np.iinfo(np.int16).max),\n                 np.int32 : (np.iinfo(np.int32).min,np.iinfo(np.int32).max),\n                 np.int64 : (np.iinfo(np.int64).min,np.iinfo(np.int64).max)\n                }\n    float_types = {np.float16: (np.finfo(np.float16).min,np.finfo(np.float16).max), \n                   np.float32: (np.finfo(np.float32).min,np.finfo(np.float32).max), \n                   np.float64: (np.finfo(np.float64).min,np.finfo(np.float64).max)\n                  }\n    for col in df.columns:\n        col_type = df[col].dtypes\n        col_min = df[col].min()\n        col_max = df[col].max()\n        if (str(col_type)[:3] == 'int') & (str(col) in categorical):\n            df[col] = pd.Categorical(df[col])\n        elif str(col_type)[:3] == 'int':\n            for dtype,drange in int_types.items():\n                if (col_min > drange[0]) & (col_max < drange[1]):\n                    df[col] = df[col].astype(dtype)\n                    break\n        elif str(col_type)[:5] == 'float':\n            for dtype,drange in float_types.items():\n                if (col_min > drange[0]) & (col_max < drange[1]):\n                    df[col] = df[col].astype(dtype)\n                    break\n    return df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Original Dataset Statistics"},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_name = ['building','train','test','weather_train','weather_test']\n\norigianl_ds = ds_statistics(dataset_name,[building_df,train_df,test_df,weather_train_df,weather_test_df])\nprint(origianl_ds.head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Optimized Datasets Statistics"},{"metadata":{"trusted":true},"cell_type":"code","source":"optimized_ds = ds_statistics(dataset_name,[ds_optimization(building_df),ds_optimization(train_df,['meter']),ds_optimization(test_df),ds_optimization(weather_train_df),ds_optimization(weather_test_df)])\nprint(optimized_ds.head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Datasets Memory Changes (%)"},{"metadata":{"trusted":true},"cell_type":"code","source":"optimized_percentage = 100 * (origianl_ds['Size (MB)'] - optimized_ds['Size (MB)'])/origianl_ds['Size (MB)']\nprint(optimized_percentage)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualize Memory Reduction"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 5))  # width:20, height:3\nplt.title('Datasets Memory Decreased (%)')\nplt.xlabel('Dataset')\nplt.ylim([0,100])\nplt.ylabel('% Changes')\nplt.bar(optimized_percentage.index,optimized_percentage.values,align='center')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Please upvote for this bit of help and motivate me.**"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}