{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":7391,"databundleVersionId":44328,"sourceType":"competition"}],"dockerImageVersionId":30170,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\n!pip install pyunpack\n!pip install patool\nos.system('apt-get install p7zip')\n\nfrom pyunpack import Archive\nimport shutil\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-08-03T22:28:08.327686Z","iopub.execute_input":"2024-08-03T22:28:08.328098Z"},"trusted":true},"execution_count":null,"outputs":[{"name":"stdout","text":"Collecting pyunpack\n  Downloading pyunpack-0.3-py2.py3-none-any.whl (4.1 kB)\nCollecting easyprocess\n  Downloading EasyProcess-1.1-py3-none-any.whl (8.7 kB)\nCollecting entrypoint2\n  Downloading entrypoint2-1.1-py2.py3-none-any.whl (9.9 kB)\n","output_type":"stream"}]},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        Archive(os.path.join(dirname, filename)).extractall('/kaggle/working/')\n        print(os.path.join(dirname, filename))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \n# optimize memory loading by specifying the data types...\ntypes = {'id': 'int64',\n         'item_nbr': 'int32',\n         'store_nbr': 'int8',\n         'unit_sales': 'float32',\n         'onpromotion': 'object'}\n%time df_all = pd.read_csv('./train.csv', parse_dates=['date'],  dtype=types, infer_datetime_format=True)\ndf_all.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* If you set low_memory=False, it will run out of memory regardless of how much memory you have.\n* In order to limit the amount of space that it takes up when you read in, we create a dictionary for each column name to the data type of that column. It is up to you to figure out the data types by running or less or head on the dataset.\n* With these tweaks, we can read in 125,497,040 rows in less than 2 minutes.\n","metadata":{"execution":{"iopub.status.busy":"2022-03-17T05:10:36.312489Z","iopub.execute_input":"2022-03-17T05:10:36.312797Z","iopub.status.idle":"2022-03-17T05:10:36.367458Z","shell.execute_reply.started":"2022-03-17T05:10:36.31275Z","shell.execute_reply":"2022-03-17T05:10:36.366526Z"},"jupyter":{"outputs_hidden":true}}}]}