{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Competition Citation\n\n@misc{child-mind-institute-detect-sleep-states,\n\n    author = {Greg Kiar, Maggie, Nathalia Esper, Ryan Holbrook, Yuki Kotani},\n    \n    title = {Child Mind Institute - Detect Sleep States},\n    \n    publisher = {Kaggle},\n    \n    year = {2023},\n    \n    url = {https://kaggle.com/competitions/child-mind-institute-detect-sleep-states}\n}","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"# References \n\n1. [ Pandas Documentation | Scaling to large datasets | https://pandas.pydata.org/docs/user_guide/scale.html ](https://pandas.pydata.org/docs/user_guide/scale.html)\n2. [Zzz💤: good night sleep with 80% Memory Reduction | by andradaolteanu | https://www.kaggle.com/code/andradaolteanu/zzz-good-night-sleep-with-80-memory-reduction ](https://www.kaggle.com/code/andradaolteanu/zzz-good-night-sleep-with-80-memory-reduction)","metadata":{}},{"cell_type":"markdown","source":"# 20 Lines of code to save 94 % Memory for train_events ","metadata":{}},{"cell_type":"code","source":"import pandas as pd\ntrain_events = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")\ntrain_events_mem_before , dtypes_before = train_events.memory_usage(deep=True) ,train_events.dtypes\ntrain_events = train_events.dropna().reset_index(drop = True)\ntrain_events[\"series_id\"] = train_events[\"series_id\"].astype(\"category\")\ntrain_events[\"night\"] = pd.to_numeric(train_events[\"night\"], downcast=\"unsigned\")\ntrain_events[\"step\"] = pd.to_numeric(train_events[\"step\"], downcast=\"unsigned\")\ntrain_events[\"event\"] = train_events[\"event\"].map({\"onset\":1,\"wakeup\":0})\ntrain_events[\"event\"] = pd.to_numeric(train_events[\"event\"], downcast=\"unsigned\")\ntrain_events[\"timestamp\"] = pd.to_datetime(train_events[\"timestamp\"],errors = \"coerce\",format='ISO8601')\ntrain_events[\"timestamp\"] = pd.to_datetime(train_events[\"timestamp\"],errors = \"coerce\",format='ISO8601')\ntrain_events['timestamp'] = train_events['timestamp'].dt.tz_localize(None)\ntrain_events_mem_after , dtypes_after = train_events.memory_usage(deep=True) ,train_events.dtypes\nprint(\"Data Types before Optimization :\\n\\n\",dtypes_before,end = \"\\n\\n\\n###\\n\\n\\n\")\nprint(\"Data Types after Optimization :\\n\\n\",dtypes_after,end = \"\\n\\n\\n###\\n\\n\\n\")\nprint(\"Memory Usage before Optimization :\\n\\n\",train_events_mem_before,end = \"\\n\\n\\n###\\n\\n\\n\")\nprint(\"Memory Usage after Optimization :\\n\\n\",train_events_mem_after,end = \"\\n\\n\\n###\\n\\n\\n\")\nprint(\"Columnwise Memory Reduction : \\n\",round (100 * (1 -  (train_events_mem_after/train_events_mem_before) ), 2 ),end = \"\\n\\n\\n###\\n\\n\\n\")\nprint(\"Memory reduction from {:.2f} mb to {:.2f} mb\".format(train_events_mem_before.sum()/(1024*1024),train_events_mem_after.sum() /(1024*1024)), end = \"\\n\\n\\n###\\n\\n\\n\")\nprint(\"Overall Memory Reduction : \",round (100 * (1 -  (train_events_mem_after.sum()/train_events_mem_before.sum()) ), 2 ), \"%\",end = \"\\n\\n\\n###\\n\\n\\n\")","metadata":{"execution":{"iopub.status.busy":"2023-09-23T06:43:56.726248Z","iopub.execute_input":"2023-09-23T06:43:56.726712Z","iopub.status.idle":"2023-09-23T06:43:57.074552Z","shell.execute_reply.started":"2023-09-23T06:43:56.726676Z","shell.execute_reply":"2023-09-23T06:43:57.073454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 14 Lines of code to save 86 % Memory for train_series.parquet","metadata":{"execution":{"iopub.status.busy":"2023-09-23T06:38:17.025860Z","iopub.execute_input":"2023-09-23T06:38:17.026265Z","iopub.status.idle":"2023-09-23T06:38:17.031084Z","shell.execute_reply.started":"2023-09-23T06:38:17.026229Z","shell.execute_reply":"2023-09-23T06:38:17.030014Z"}}},{"cell_type":"code","source":"train_series = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\")\ntrain_series_mem_before , dtypes_before = train_series.memory_usage(deep=True) ,train_series.dtypes\ntrain_series = train_series.dropna().reset_index(drop = True)\ntrain_series[\"series_id\"] = train_series[\"series_id\"].astype(\"category\")\ntrain_series[\"step\"] = pd.to_numeric(train_series[\"step\"], downcast=\"unsigned\")\ntrain_series[\"timestamp\"] = train_series.groupby(\"series_id\")[\"timestamp\"].transform(lambda x: pd.to_datetime(pd.to_datetime(x, errors = \"coerce\"), errors = \"coerce\").dt.tz_localize(None))\ntrain_series_mem_after , dtypes_after = train_series.memory_usage(deep=True) ,train_series.dtypes\nprint(\"Data Types before Optimization :\\n\\n\",dtypes_before,end = \"\\n\\n\\n###\\n\\n\\n\")\nprint(\"Data Types after Optimization :\\n\\n\",dtypes_after,end = \"\\n\\n\\n###\\n\\n\\n\")\nprint(\"Memory Usage before Optimization :\\n\\n\",train_series_mem_before,end = \"\\n\\n\\n###\\n\\n\\n\")\nprint(\"Memory Usage after Optimization :\\n\\n\",train_series_mem_after,end = \"\\n\\n\\n###\\n\\n\\n\")\nprint(\"Columnwise Memory Reduction : \\n\",round (100 * (1 -  (train_series_mem_after/train_series_mem_before) ), 2 ),end = \"\\n\\n\\n###\\n\\n\\n\")\nprint(\"Memory reduction from {:.2f} mb to {:.2f} mb\".format(train_series_mem_before.sum()/(1024*1024),train_series_mem_after.sum() /(1024*1024)), end = \"\\n\\n\\n###\\n\\n\\n\")\nprint(\"Overall Memory Reduction : \",round (100 * (1 -  (train_series_mem_after.sum()/train_series_mem_before.sum()) ), 2 ), \"%\",end = \"\\n\\n\\n###\\n\\n\\n\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Keep Watching this space for any updates\n* Please let me know your feedback and/or comments","metadata":{}}]}