{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input/favorita-grocery-sales-forecasting'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"f9c4f449-b48a-40d4-b9e2-7c4f4f854b56","_cell_guid":"22b6f77e-9b5d-4359-81a0-0773fd6610c4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-05T07:33:28.655760Z","iopub.execute_input":"2023-06-05T07:33:28.656279Z","iopub.status.idle":"2023-06-05T07:33:28.689177Z","shell.execute_reply.started":"2023-06-05T07:33:28.656173Z","shell.execute_reply":"2023-06-05T07:33:28.688524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install py7zr","metadata":{"_uuid":"aa163cf5-2558-4da1-b986-39b690e6581f","_cell_guid":"447cc402-6805-40d9-b42b-9cdd0d9ac0bc","collapsed":false,"_kg_hide-output":true,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-05T07:33:32.174074Z","iopub.execute_input":"2023-06-05T07:33:32.174436Z","iopub.status.idle":"2023-06-05T07:33:45.550510Z","shell.execute_reply.started":"2023-06-05T07:33:32.174387Z","shell.execute_reply":"2023-06-05T07:33:45.549531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import py7zr\nfrom subprocess import check_output\n\nfor dirname, _, filenames in os.walk('/kaggle/input/favorita-grocery-sales-forecasting'):\n    for filename in filenames:\n        archive = py7zr.SevenZipFile(os.path.join(dirname, filename), mode='r')\n        archive.extractall(path=\"/kaggle/working\")\n        archive.close()\n\nprint(check_output([\"ls\", \"../working\"]).decode(\"utf8\"))","metadata":{"_uuid":"539b8ff1-c749-4ba0-85b8-0b6ff72e3b01","_cell_guid":"d4d7ace8-d0c5-4aed-bb1d-a6edbaee8076","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-05T07:35:11.632822Z","iopub.execute_input":"2023-06-05T07:35:11.633083Z","iopub.status.idle":"2023-06-05T07:36:32.003341Z","shell.execute_reply.started":"2023-06-05T07:35:11.633051Z","shell.execute_reply":"2023-06-05T07:36:32.001632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{"_uuid":"4a10af6b-20ae-4f86-a548-07baffa77ac7","_cell_guid":"2e8d8197-afcc-4ba3-b275-dfdc21521b17","trusted":true}},{"cell_type":"code","source":"!pip install pyunpack\n!pip install wget","metadata":{"execution":{"iopub.status.busy":"2023-06-05T07:36:32.005564Z","iopub.execute_input":"2023-06-05T07:36:32.005889Z","iopub.status.idle":"2023-06-05T07:36:52.154354Z","shell.execute_reply.started":"2023-06-05T07:36:32.005861Z","shell.execute_reply":"2023-06-05T07:36:52.152834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport glob\nimport os\nimport shutil\nimport sys\n\n# from expt_settings.configs import ExperimentConfig\nimport numpy as np\nimport pandas as pd\nimport pyunpack\nimport wget","metadata":{"_uuid":"d25b35a0-57a5-457c-bee2-c8401d467746","_cell_guid":"c2ea31c2-627c-4a98-9d72-6b3c9e392cc6","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-05T07:36:58.168905Z","iopub.execute_input":"2023-06-05T07:36:58.169157Z","iopub.status.idle":"2023-06-05T07:36:58.177124Z","shell.execute_reply.started":"2023-06-05T07:36:58.169126Z","shell.execute_reply":"2023-06-05T07:36:58.176021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Explore and prepare the data","metadata":{"_uuid":"31e02bf3-724c-41c7-93a9-a8ff100bd6a6","_cell_guid":"99c92338-b64b-473e-a952-aba4597beebb","trusted":true}},{"cell_type":"code","source":"items = pd.read_csv(\"../working/items.csv\")\nholiday_events = pd.read_csv(\"../working/holidays_events.csv\", parse_dates=['date'])\nstores = pd.read_csv(\"../working/stores.csv\")\noil = pd.read_csv(\"../working/oil.csv\", parse_dates=['date'])\ntransactions = pd.read_csv(\"../working/transactions.csv\", parse_dates=['date'])\n# the full training data's output: \"125,497,040 rows | 6 columns\"\n#Therefore I will only load approx 5% of the data just to get a rough idea of what is in store for us.\ntemporal = pd.read_csv(\"../working/train.csv\", nrows=6000  , parse_dates=['date'])\n# train_large = pd.read_csv('../working/train.csv', skiprows = 115000000, names = train.columns, parse_dates = ['date'])","metadata":{"_uuid":"991c10b6-a90a-4910-9308-9623cc2b7193","_cell_guid":"a3a0f3a7-af50-4d9d-a212-448d0566b110","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-05T07:47:51.014620Z","iopub.execute_input":"2023-06-05T07:47:51.015409Z","iopub.status.idle":"2023-06-05T07:47:51.083241Z","shell.execute_reply.started":"2023-06-05T07:47:51.015367Z","shell.execute_reply":"2023-06-05T07:47:51.082147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temporal['date'] = pd.to_datetime(temporal['date'])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T07:47:53.536731Z","iopub.execute_input":"2023-06-05T07:47:53.537340Z","iopub.status.idle":"2023-06-05T07:47:53.553205Z","shell.execute_reply.started":"2023-06-05T07:47:53.537246Z","shell.execute_reply":"2023-06-05T07:47:53.551921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dates = temporal['date'].unique()\n\n# Add trajectory identifier\ntemporal['traj_id'] = temporal['store_nbr'].apply(str) + '_' + temporal['item_nbr'].apply(str)\ntemporal['unique_id'] = temporal['traj_id'] + '_' + temporal['date'].apply(str)\n\n# Remove all IDs with negative returns\nprint('Removing returns data')\nmin_returns = temporal['unit_sales'].groupby(temporal['traj_id']).min()\nvalid_ids = set(min_returns[min_returns >= 0].index)\nselector = temporal['traj_id'].apply(lambda traj_id: traj_id in valid_ids)\nnew_temporal = temporal[selector].copy()\ndel temporal\ngc.collect()\ntemporal = new_temporal\ntemporal['open'] = 1\n\n# Resampling\nprint('Resampling to regular grid')\nresampled_dfs = []\nfor traj_id, raw_sub_df in temporal.groupby('traj_id'):\n    print('Resampling', traj_id)\n    sub_df = raw_sub_df.set_index('date', drop=True).copy()\n    sub_df = sub_df.resample('1d').last()\n    sub_df['date'] = sub_df.index\n    sub_df[['store_nbr', 'item_nbr', 'onpromotion']] \\\n        = sub_df[['store_nbr', 'item_nbr', 'onpromotion']].fillna(method='ffill')\n    sub_df['open'] = sub_df['open'].fillna(\n        0)  # flag where sales data is unknown\n    sub_df['log_sales'] = np.log(sub_df['unit_sales'])\n\n    resampled_dfs.append(sub_df.reset_index(drop=True))\n\nnew_temporal = pd.concat(resampled_dfs, axis=0)\ndel temporal\ngc.collect()\ntemporal = new_temporal\n\nprint('Adding oil')\noil.name = 'oil'\noil.index = pd.to_datetime(oil.index)\ntemporal = temporal.join(\n    oil.loc[dates].fillna(method='ffill'), on='date', how='left')\ntemporal['oil'] = temporal['oil'].fillna(-1)\n\nprint('Adding store info')\ntemporal = temporal.join(store_info, on='store_nbr', how='left')\n\nprint('Adding item info')\ntemporal = temporal.join(items, on='item_nbr', how='left')\n\ntransactions['date'] = pd.to_datetime(transactions['date'])\ntemporal = temporal.merge(\n  transactions,\n  left_on=['date', 'store_nbr'],\n  right_on=['date', 'store_nbr'],\n  how='left')\ntemporal['transactions'] = temporal['transactions'].fillna(-1)\n\n# Additional date info\ntemporal['day_of_week'] = pd.to_datetime(temporal['date'].values).dayofweek\ntemporal['day_of_month'] = pd.to_datetime(temporal['date'].values).day\ntemporal['month'] = pd.to_datetime(temporal['date'].values).month\n\n# Add holiday info\nprint('Adding holidays')\nholiday_subset = holidays[holidays['transferred'].apply(\n  lambda x: not x)].copy()\nholiday_subset.columns = [\n  s if s != 'type' else 'holiday_type' for s in holiday_subset.columns\n]\nholiday_subset['date'] = pd.to_datetime(holiday_subset['date'])\nlocal_holidays = holiday_subset[holiday_subset['locale'] == 'Local']\nregional_holidays = holiday_subset[holiday_subset['locale'] == 'Regional']\nnational_holidays = holiday_subset[holiday_subset['locale'] == 'National']\n\ntemporal['national_hol'] = temporal.merge(\n  national_holidays, left_on=['date'], right_on=['date'],\n  how='left')['description'].fillna('')\ntemporal['regional_hol'] = temporal.merge(\n  regional_holidays,\n  left_on=['state', 'date'],\n  right_on=['locale_name', 'date'],\n  how='left')['description'].fillna('')\ntemporal['local_hol'] = temporal.merge(\n  local_holidays,\n  left_on=['city', 'date'],\n  right_on=['locale_name', 'date'],\n  how='left')['description'].fillna('')\n\ntemporal.sort_values('unique_id', inplace=True)\n\nprint('Saving processed file to {}'.format(config.data_csv_path))\ntemporal.to_csv(config.data_csv_path)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-05T07:47:57.010255Z","iopub.execute_input":"2023-06-05T07:47:57.010738Z","iopub.status.idle":"2023-06-05T07:48:49.490756Z","shell.execute_reply.started":"2023-06-05T07:47:57.010690Z","shell.execute_reply":"2023-06-05T07:48:49.489081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}