{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd \nimport numpy as np \nimport os\ndir_data = \"../input/big-data-derby-2022/nyra_2019_complete.csv\"\ndata_size = os.path.getsize(dir_data)/1e9\nprint(f\"The file size is {data_size:.4f} GB\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-27T04:42:04.288891Z","iopub.execute_input":"2022-09-27T04:42:04.289336Z","iopub.status.idle":"2022-09-27T04:42:04.300358Z","shell.execute_reply.started":"2022-09-27T04:42:04.289298Z","shell.execute_reply":"2022-09-27T04:42:04.299504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/code/gemartin/load-data-reduce-memory-usage/notebook\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df\n\n\ndef import_data(file):\n    \"\"\"create a dataframe and optimize its memory usage\"\"\"\n    df = pd.read_csv(file, parse_dates=True, keep_date_col=True)\n    df = reduce_mem_usage(df)\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-09-27T04:42:04.658006Z","iopub.execute_input":"2022-09-27T04:42:04.658861Z","iopub.status.idle":"2022-09-27T04:42:04.674940Z","shell.execute_reply.started":"2022-09-27T04:42:04.658812Z","shell.execute_reply":"2022-09-27T04:42:04.673998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('-' * 80)\nprint('data')\nbig_data_derby = import_data(dir_data)","metadata":{"execution":{"iopub.status.busy":"2022-09-27T04:42:05.034904Z","iopub.execute_input":"2022-09-27T04:42:05.035319Z","iopub.status.idle":"2022-09-27T04:42:20.565600Z","shell.execute_reply.started":"2022-09-27T04:42:05.035283Z","shell.execute_reply":"2022-09-27T04:42:20.564352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> You can see we compressed the memory usage of the dataframe from 718+ MB to 154.59 MB without losing some information. Then you can load as usual by using pandas.","metadata":{}}]}