{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Implementation of Pandas DF memory reduction","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom pandas.api.types import is_datetime64_ns_dtype","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-09-12T23:46:44.027487Z","iopub.execute_input":"2023-09-12T23:46:44.027934Z","iopub.status.idle":"2023-09-12T23:46:44.034072Z","shell.execute_reply.started":"2023-09-12T23:46:44.027901Z","shell.execute_reply":"2023-09-12T23:46:44.032626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \n    \"\"\" \n    Iterate through all numeric columns of a dataframe and modify the data type\n    to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage of dataframe is {start_mem:.2f} MB')\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object and not is_datetime64_ns_dtype(df[col]):\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float32)\n        \n    df['series_id'] = df['series_id'].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(f'Memory usage after optimization is: {end_mem:.2f} MB')\n    decrease = 100 * (start_mem - end_mem) / start_mem\n    print(f'Decreased by {decrease:.2f}%')\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2023-09-12T23:46:44.410641Z","iopub.execute_input":"2023-09-12T23:46:44.411045Z","iopub.status.idle":"2023-09-12T23:46:44.424639Z","shell.execute_reply.started":"2023-09-12T23:46:44.411015Z","shell.execute_reply":"2023-09-12T23:46:44.423444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE = '/kaggle/input/child-mind-institute-detect-sleep-states/'","metadata":{"execution":{"iopub.status.busy":"2023-09-12T23:46:45.417612Z","iopub.execute_input":"2023-09-12T23:46:45.418032Z","iopub.status.idle":"2023-09-12T23:46:45.422735Z","shell.execute_reply.started":"2023-09-12T23:46:45.417992Z","shell.execute_reply":"2023-09-12T23:46:45.421561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series = pd.read_parquet(BASE+'train_series.parquet')\ntest_series = pd.read_parquet(BASE+'test_series.parquet')\nsample_submission = pd.read_csv(BASE+'sample_submission.csv')\ntrain_events = pd.read_csv(BASE+'train_events.csv')","metadata":{"execution":{"iopub.status.busy":"2023-09-12T23:46:46.465332Z","iopub.execute_input":"2023-09-12T23:46:46.465773Z","iopub.status.idle":"2023-09-12T23:47:41.576901Z","shell.execute_reply.started":"2023-09-12T23:46:46.465742Z","shell.execute_reply":"2023-09-12T23:47:41.575851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series = reduce_mem_usage(train_series)","metadata":{"execution":{"iopub.status.busy":"2023-09-12T23:47:41.579473Z","iopub.execute_input":"2023-09-12T23:47:41.579943Z","iopub.status.idle":"2023-09-12T23:47:56.322718Z","shell.execute_reply.started":"2023-09-12T23:47:41.579901Z","shell.execute_reply":"2023-09-12T23:47:56.321554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_series = reduce_mem_usage(test_series)","metadata":{"execution":{"iopub.status.busy":"2023-09-12T23:47:56.324127Z","iopub.execute_input":"2023-09-12T23:47:56.324445Z","iopub.status.idle":"2023-09-12T23:47:56.335070Z","shell.execute_reply.started":"2023-09-12T23:47:56.324418Z","shell.execute_reply":"2023-09-12T23:47:56.333885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = reduce_mem_usage(train_events)","metadata":{"execution":{"iopub.status.busy":"2023-09-12T23:47:56.337347Z","iopub.execute_input":"2023-09-12T23:47:56.337653Z","iopub.status.idle":"2023-09-12T23:47:56.357061Z","shell.execute_reply.started":"2023-09-12T23:47:56.337627Z","shell.execute_reply":"2023-09-12T23:47:56.355636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###### ","metadata":{}}]}