{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1b20b53179e2ac8fe205f73eb6a7db10d768e00f"},"cell_type":"markdown","source":"I see a lot of kernels using this particular function to reduce the dataset memory usage.\n\nWhile it's very useful (props to the person that wrote it!) it makes usage of the float16 data type.\n\nThis data type has only 1e-3 precison and some features (including the target) have more significative figures than that."},{"metadata":{"trusted":true,"_uuid":"eb7fee3608c9b945ad1095386aab40c22dfc39d5"},"cell_type":"code","source":"np.finfo(np.float16)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"040f6b322185ea4c28008ba65eb2404cab78afb3"},"cell_type":"markdown","source":"\nWhile i don't think it would be that big of an issue in a real scenario, I do think it could make a difference in a kaggle competition.\n\nHere I wrote a little example:"},{"metadata":{"trusted":true,"_uuid":"266466d3df7c06643750546a2e3c15c8aa8d4cc4"},"cell_type":"code","source":"cards = pd.read_csv(\"../input/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16) \n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64) \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e44889b7f321096caac8d13fc744f77112eedbb0"},"cell_type":"code","source":"cards.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8f894a64d8936ebebdae5ddb262882ae020bcf2"},"cell_type":"code","source":"cards_degraded = reduce_mem_usage(cards.copy())\ncards_degraded.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"43a18ef5a452d6eab3e49994f084648dfe3f1df7"},"cell_type":"markdown","source":"As you can see, target values have change.\n\nYou can avoid this problem by ditching float16 dtype"},{"metadata":{"trusted":true,"_uuid":"4b4850ce62d2a9714b67d7c7cdada957d6b11e85"},"cell_type":"code","source":"def reduce_mem_usage_safer(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64) \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4308da7cb088d0db78b36ffe4bb641309ac8652"},"cell_type":"code","source":"cards.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d54a4fecaafc46b3f71a5e02e8b8e6df88e1f2a5"},"cell_type":"code","source":"cards_less_degraded = reduce_mem_usage_safer(cards.copy())\ncards_less_degraded.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cf96cc710ea7acd72bb7ae0d98c07f92f3818f27"},"cell_type":"markdown","source":"Hope It's useful!"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}