{"cells":[{"metadata":{},"cell_type":"markdown","source":"This is an extension of this [data minification notebook](https://www.kaggle.com/kyakovlev/ashrae-data-minification). Minification is one of the first steps that should be done as it can:\n\n* Load data faster -> pickle format (best choice for me)\n* Consume less memory -> Carefull types downcasting\n* Avoid repeated cleaning steps.\n\n**Possible problems**:\n\n* pandas pickle format is \"version dependent\" and you may have problems with loading data generated with a newer version of Pandas\n* Any bad transformation may and will affect your model. Minification should be done very veeeeery careful as you will use it in every single your kernel in the future. If you not sure about downcasting some column -> don't do. If not sure about cleaning -> don't do. All wrong decisions here may and will affect your score. You may spend days searching for error that prevents your model converge well and find that it was you who created an error in the minification step."},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os, gc, sys, warnings, random, math, psutil, pickle\n\nfrom sklearn.preprocessing import LabelEncoder\n\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Load Data')\ntrain_df = pd.read_csv('../input/ashrae-energy-prediction/train.csv')\ntest_df = pd.read_csv('../input/ashrae-energy-prediction/test.csv')\n\nbuilding_df = pd.read_csv('../input/ashrae-energy-prediction/building_metadata.csv')\n\ntrain_weather_df = pd.read_csv('../input/ashrae-energy-prediction/weather_train.csv')\ntest_weather_df = pd.read_csv('../input/ashrae-energy-prediction/weather_test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"########################### Convert Timestamp to Date\n#################################################################################\nfor df in [train_df, test_df, train_weather_df, test_weather_df]:\n    df['timestamp'] = pd.to_datetime(df['timestamp'])\n    \nfor df in [train_df, test_df]:\n    df['DT_Y'] = df['timestamp'].dt.year-2000\n    df['DT_M'] = df['timestamp'].dt.month.astype(np.int8)\n    df['DT_W'] = df['timestamp'].dt.weekofyear.astype(np.int8)\n    df['DT_D'] = df['timestamp'].dt.dayofyear.astype(np.int16)\n    df['DT_hour'] = df['timestamp'].dt.hour.astype(np.int8)\n    df['DT_dayofweek'] = df['timestamp'].dt.dayofweek.astype(np.int8)\n    df['DT_day_month'] = df['timestamp'].dt.day.astype(np.int8)\n    df['DT_week_month'] = df['timestamp'].dt.day/7\n    df['DT_week_month'] = df['DT_week_month'].apply(lambda x: math.ceil(x)).astype(np.int8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"########################### Strings to Category\n#################################################################################\nbuilding_df['primary_use'] = building_df['primary_use'].astype('category')\n\n########################### Building Transform\n#################################################################################\nbuilding_df['floor_count'] = building_df['floor_count'].fillna(building_df['floor_count'].dropna().median()).astype(np.int8)\nbuilding_df['year_built'] = building_df['year_built'].fillna(building_df['year_built'].dropna().median()).astype(np.int16)\n\nle = LabelEncoder()\nbuilding_df['primary_use'] = building_df['primary_use'].astype(str)\nbuilding_df['primary_use'] = le.fit_transform(building_df['primary_use']).astype(np.int8)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"########################### Helpers\n#################################################################################\n## -------------------\n## Memory Reducer\n# :df pandas dataframe to reduce size             # type: pd.DataFrame()\n# :verbose                                        # type: bool\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n## -------------------","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"########################### Convert to Reduece Memory\n#################################################################################\ndo_not_convert = ['category','datetime64[ns]','object']\nfor df in [train_df, test_df, building_df, train_weather_df, test_weather_df]:\n    original = df.copy()\n    df = reduce_mem_usage(df)\n\n    for col in list(df):\n        if df[col].dtype.name not in do_not_convert:\n            if (df[col]-original[col]).sum()!=0:\n                df[col] = original[col]\n                print('Bad transformation', col)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"########################### Data Check\n#################################################################################\nprint('Main data:', list(train_df), train_df.info())\nprint('#'*20)\n\nprint('Buildings data:',list(building_df), building_df.info())\nprint('#'*20)\n\nprint('Weather data:',list(train_weather_df), train_weather_df.info())\nprint('#'*20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"########################### Building DF merge through concat \n#################################################################################\n# Benefits of concat:\n## Faster for huge datasets (columns number)\n## No dtype change for dataset\n## Consume less memmory \n\ntemp_df = train_df[['building_id']]\ntemp_df = temp_df.merge(building_df, on=['building_id'], how='left')\ndel temp_df['building_id']\ntrain_df = pd.concat([train_df, temp_df], axis=1)\n\ntemp_df = test_df[['building_id']]\ntemp_df = temp_df.merge(building_df, on=['building_id'], how='left')\ndel temp_df['building_id']\ntest_df = pd.concat([test_df, temp_df], axis=1)\n\ndel building_df, temp_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"########################### Weather DF merge over concat (to not lose type)\n#################################################################################\n# Benefits of concat:\n## Faster for huge datasets (columns number)\n## No dtype change for dataset\n## Consume less memmory \n\ntemp_df = train_df[['site_id','timestamp']]\ntemp_df = temp_df.merge(train_weather_df, on=['site_id','timestamp'], how='left')\ndel temp_df['site_id'], temp_df['timestamp']\ntrain_df = pd.concat([train_df, temp_df], axis=1)\n\ntemp_df = test_df[['site_id','timestamp']]\ntemp_df = temp_df.merge(test_weather_df, on=['site_id','timestamp'], how='left')\ndel temp_df['site_id'], temp_df['timestamp']\ntest_df = pd.concat([test_df, temp_df], axis=1)\n\ndel train_weather_df, test_weather_df, temp_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def average_imputation(df, column_name):\n    imputation = df.groupby(['timestamp'])[column_name].mean()\n    \n    df.loc[df[column_name].isnull(), column_name] = df[df[column_name].isnull()][[column_name]].apply(lambda x: imputation[df['timestamp'][x.index]].values)\n    del imputation\n    return df\n\nbeaufort = [(0, 0, 0.3), (1, 0.3, 1.6), (2, 1.6, 3.4), (3, 3.4, 5.5), (4, 5.5, 8), (5, 8, 10.8), (6, 10.8, 13.9), \n          (7, 13.9, 17.2), (8, 17.2, 20.8), (9, 20.8, 24.5), (10, 24.5, 28.5), (11, 28.5, 33), (12, 33, 200)]\n\nclass ASHRAE3Preprocessor(object):\n    @classmethod\n    def fit(cls, df):\n        data_ratios = df.count()/len(df)\n        cls.avgs = df.loc[:,data_ratios < 1.0].mean()\n\n    @classmethod\n    def transform(cls, df):\n        #df = df.fillna(cls.avgs) # refill NAN with averages\n        data_ratios = df.count()/len(df)\n        columns_to_fill = data_ratios[data_ratios < 1.0].index.values.tolist()\n        for col in columns_to_fill:\n            df = average_imputation(df, col)\n        \n        for item in beaufort:\n            df.loc[(df['wind_speed']>=item[1]) & (df['wind_speed']<item[2]), 'beaufort_scale'] = item[0]\n\n        # parse and cast columns to a smaller type\n        df.rename(columns={\"square_feet\": \"log_square_feet\"}, inplace=True)\n        df['log_square_feet'] = np.float16(np.log(df['log_square_feet']))\n        df['year_built'] = np.uint8(df['year_built']-1900)\n        \n        # remove redundant columns\n        for col in df.columns:\n            if col in ['timestamp', 'row_id', 'wind_speed']:\n                del df[col]\n    \n        # extract target column\n        if 'meter_reading' in df.columns:\n            df['meter_reading'] = np.log1p(df['meter_reading']).astype(np.float32) # comp metric uses log errors\n            # maybe remove some of the high outliers because of sensor error\n            df[\"meter_reading\"] = df[\"meter_reading\"].clip(upper = df[\"meter_reading\"].quantile(.999))\n\n        return df\n        \n#ASHRAE3Preprocessor.fit(train_df)\ntrain_df = ASHRAE3Preprocessor.transform(train_df)\ntest_df = ASHRAE3Preprocessor.transform(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"########################### Trick to use kernel hdd to store results\n#################################################################################\ntrain_df.to_pickle('train_df.pkl')\ntest_df.to_pickle('test_df.pkl')\n   \ndel train_df, test_df\ngc.collect()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}