{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":7391,"databundleVersionId":44328,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport seaborn as sns\nimport os\nimport pickle\nfrom collections import Counter\n\n#sns.set_style('whitegrid')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:25:12.544008Z","iopub.execute_input":"2025-06-03T04:25:12.544706Z","iopub.status.idle":"2025-06-03T04:25:14.480287Z","shell.execute_reply.started":"2025-06-03T04:25:12.544672Z","shell.execute_reply":"2025-06-03T04:25:14.479567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%capture\n!pip install recommenders\n!pip install py7zr\n!pip install lightfm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:25:15.877215Z","iopub.execute_input":"2025-06-03T04:25:15.878255Z","iopub.status.idle":"2025-06-03T04:25:56.383615Z","shell.execute_reply.started":"2025-06-03T04:25:15.878229Z","shell.execute_reply":"2025-06-03T04:25:56.382576Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import py7zr\n\nfrom sklearn.preprocessing import minmax_scale, MultiLabelBinarizer\n\nfrom recommenders.datasets.python_splitters import python_chrono_split, python_stratified_split\nfrom recommenders.utils.timer import Timer\nfrom recommenders.utils.python_utils import binarize\nfrom recommenders.models.sar import SAR\nfrom recommenders.datasets.spark_splitters import spark_random_split\nfrom pyspark.ml.recommendation import ALS\nfrom pyspark.sql import SparkSession\nfrom recommenders.evaluation.python_evaluation import precision_at_k, recall_at_k, map_at_k, ndcg_at_k\n\nfrom lightfm import LightFM\nfrom lightfm.data import Dataset","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:25:56.385256Z","iopub.execute_input":"2025-06-03T04:25:56.385612Z","iopub.status.idle":"2025-06-03T04:25:57.373464Z","shell.execute_reply.started":"2025-06-03T04:25:56.385584Z","shell.execute_reply":"2025-06-03T04:25:57.372663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for dir, _, names in os.walk('/kaggle/input/favorita-grocery-sales-forecasting'):\n    for f in names:\n        arch = py7zr.SevenZipFile(os.path.join(dir, f), mode = 'r')\n        arch.extractall(path = '/kaggle/working')\n        arch.close()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:25:57.374365Z","iopub.execute_input":"2025-06-03T04:25:57.375229Z","iopub.status.idle":"2025-06-03T04:27:15.893074Z","shell.execute_reply.started":"2025-06-03T04:25:57.375208Z","shell.execute_reply":"2025-06-03T04:27:15.892511Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Exploration and Analysis","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/working/train.csv')\ntest_df = pd.read_csv('/kaggle/working/test.csv')\nstores_df = pd.read_csv('/kaggle/working/stores.csv')\nitems_df = pd.read_csv('/kaggle/working/items.csv')\ntrans_df = pd.read_csv('/kaggle/working/transactions.csv')\nevents_df = pd.read_csv('/kaggle/working/holidays_events.csv')\noil_df = pd.read_csv('/kaggle/working/oil.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:54:40.220997Z","iopub.execute_input":"2025-06-02T22:54:40.221208Z","iopub.status.idle":"2025-06-02T22:55:55.969832Z","shell.execute_reply.started":"2025-06-02T22:54:40.221192Z","shell.execute_reply":"2025-06-02T22:55:55.969278Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"oil_df['date'] = pd.to_datetime(oil_df['date'])\noil_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:55:55.970571Z","iopub.execute_input":"2025-06-02T22:55:55.970817Z","iopub.status.idle":"2025-06-02T22:55:56.001235Z","shell.execute_reply.started":"2025-06-02T22:55:55.970792Z","shell.execute_reply":"2025-06-02T22:55:56.000573Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df['date'] = pd.to_datetime(train_df['date'])\ntrain_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:55:56.002023Z","iopub.execute_input":"2025-06-02T22:55:56.002275Z","iopub.status.idle":"2025-06-02T22:56:07.351167Z","shell.execute_reply.started":"2025-06-02T22:55:56.002252Z","shell.execute_reply":"2025-06-02T22:56:07.350573Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.head()\n# Inital thoughts:\n# 1. Aggregate date - unit sales (done)\n# 2. Aggregate itemnbr - unit sales to find top selling items (done)\n# 3. Aggreate storenbr - unit sales to find top selling stores\n# 4. Find longest promotion streaks (done)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:56:07.351884Z","iopub.execute_input":"2025-06-02T22:56:07.352172Z","iopub.status.idle":"2025-06-02T22:56:07.360284Z","shell.execute_reply.started":"2025-06-02T22:56:07.352147Z","shell.execute_reply":"2025-06-02T22:56:07.359776Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(train_df), len(train_df['item_nbr'].unique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:56:07.361025Z","iopub.execute_input":"2025-06-02T22:56:07.36126Z","iopub.status.idle":"2025-06-02T22:56:08.197456Z","shell.execute_reply.started":"2025-06-02T22:56:07.361245Z","shell.execute_reply":"2025-06-02T22:56:08.196882Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Assume type are: A -> Large stores, B -> Medium-sized stores, C -> Small stores, D -> Specially formatted stores\nstores_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:56:08.199626Z","iopub.execute_input":"2025-06-02T22:56:08.199844Z","iopub.status.idle":"2025-06-02T22:56:08.207762Z","shell.execute_reply.started":"2025-06-02T22:56:08.199829Z","shell.execute_reply":"2025-06-02T22:56:08.207043Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'Number of cities:\\t{len(stores_df[\"city\"].unique())}')\nprint(f'Number of states:\\t{len(stores_df[\"state\"].unique())}')\nprint(f'Number of types:\\t{len(stores_df[\"type\"].unique())}')\nprint(f'Number of clusters:\\t{len(stores_df[\"cluster\"].unique())}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:56:08.208334Z","iopub.execute_input":"2025-06-02T22:56:08.208497Z","iopub.status.idle":"2025-06-02T22:56:08.219602Z","shell.execute_reply.started":"2025-06-02T22:56:08.208485Z","shell.execute_reply":"2025-06-02T22:56:08.218918Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"items_df.head()\n#","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:56:08.220227Z","iopub.execute_input":"2025-06-02T22:56:08.220534Z","iopub.status.idle":"2025-06-02T22:56:08.237058Z","shell.execute_reply.started":"2025-06-02T22:56:08.220504Z","shell.execute_reply":"2025-06-02T22:56:08.236358Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trans_df['date'] = pd.to_datetime(trans_df['date'])\ntrans_df.head()\n# What store had the highest number of transactions? What type is it? Combine datasets","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:56:08.237733Z","iopub.execute_input":"2025-06-02T22:56:08.237926Z","iopub.status.idle":"2025-06-02T22:56:08.259183Z","shell.execute_reply.started":"2025-06-02T22:56:08.237908Z","shell.execute_reply":"2025-06-02T22:56:08.25869Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"events_df['date'] = pd.to_datetime(events_df['date'])\nevents_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:56:08.25978Z","iopub.execute_input":"2025-06-02T22:56:08.259962Z","iopub.status.idle":"2025-06-02T22:56:08.269798Z","shell.execute_reply.started":"2025-06-02T22:56:08.259948Z","shell.execute_reply":"2025-06-02T22:56:08.269098Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'Number of NaN values per dataset respectively:')\nprint(f'Train:\\t{train_df.isna().sum()}\\n')\nprint(f'Stores:\\t{stores_df.isna().sum()}\\n')\nprint(f'Items:\\t{items_df.isna().sum()}\\n')\nprint(f'Events:\\t{events_df.isna().sum()}\\n')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:56:08.270579Z","iopub.execute_input":"2025-06-02T22:56:08.270771Z","iopub.status.idle":"2025-06-02T22:56:14.457231Z","shell.execute_reply.started":"2025-06-02T22:56:08.270758Z","shell.execute_reply":"2025-06-02T22:56:14.45642Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (10, 6))\n\ndaily_sales = train_df.groupby('date')['unit_sales'].sum() # Aggregate unit sales per day\nrolling_30 = daily_sales.shift(1).rolling(30).mean()\n\nlowest_spikes = daily_sales.nsmallest(5)\n\ndaily_sales.plot(zorder = 1, alpha = 1, label = 'Unit Sales')\nrolling_30.plot(alpha = 0.9, color = 'red', label = '30 Days Rolling Average')\n\nplt.scatter(lowest_spikes.index, lowest_spikes.values + 0.01e6, marker = 'x', color = 'red', alpha = 1, zorder = 2)\n\nfor d, v in lowest_spikes.items():\n    plt.text(x = d, y = v - 0.065e6, s = str(f'{d.year}-{d.month}-{d.day}'), horizontalalignment = 'center', color = 'red')\n\nplt.xlabel('Date')\nplt.ylabel('Unit Sales')\nplt.title('Unit Sales over Time')\nplt.legend()\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:56:14.458079Z","iopub.execute_input":"2025-06-02T22:56:14.458328Z","iopub.status.idle":"2025-06-02T22:56:17.483656Z","shell.execute_reply.started":"2025-06-02T22:56:14.458311Z","shell.execute_reply":"2025-06-02T22:56:17.482935Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"item_agg = train_df.groupby('item_nbr')['unit_sales'].sum().sort_values(ascending = False)\ntop_items = item_agg.nlargest(4)\n\nitem_agg_items_df = items_df[items_df['item_nbr'].isin(item_agg.index)]\ntop_3_data = item_agg_items_df[item_agg_items_df['item_nbr'].isin(top_items.index)]['family'].unique()\n\nitem_agg_items_df['agg'] = item_agg.values\nitem_agg_items_df['hue'] = item_agg_items_df['family'].apply(lambda x: 1 if x in top_3_data else 0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:56:17.484372Z","iopub.execute_input":"2025-06-02T22:56:17.484611Z","iopub.status.idle":"2025-06-02T22:56:20.517645Z","shell.execute_reply.started":"2025-06-02T22:56:17.484595Z","shell.execute_reply":"2025-06-02T22:56:20.516889Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import textwrap # Wrap text around\n\nplt.figure(figsize = (15, 7))\nax = sns.histplot(x = item_agg_items_df['family'], kde = True, edgecolor = 'black', hue = item_agg_items_df['hue'])\nax.legend_.remove()\n\nlabels = [textwrap.fill(label.get_text(), 12) for label in ax.get_xticklabels()]\nax.set_xticklabels(labels);\n\nplt.xticks(rotation = 45, fontsize = 8)\nplt.xlabel('Item Category')\nplt.ylabel('Item Frequency')\nplt.title('Items Distribution')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:56:20.518282Z","iopub.execute_input":"2025-06-02T22:56:20.51854Z","iopub.status.idle":"2025-06-02T22:56:20.955663Z","shell.execute_reply.started":"2025-06-02T22:56:20.518501Z","shell.execute_reply":"2025-06-02T22:56:20.95496Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_promo = train_df.sort_values(['item_nbr', 'store_nbr', 'date'])\ndf_promo['onpromotion'] = df_promo['onpromotion'].fillna(False).astype(bool)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:56:20.956569Z","iopub.execute_input":"2025-06-02T22:56:20.956866Z","iopub.status.idle":"2025-06-02T22:57:04.20896Z","shell.execute_reply.started":"2025-06-02T22:56:20.956843Z","shell.execute_reply":"2025-06-02T22:57:04.208362Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Find where promotion streak breaks.\ndf_promo['promo_break'] = (~df_promo['onpromotion']).groupby([df_promo['item_nbr'], df_promo['store_nbr']]).cumsum()\ndf_promo['consec_promo_days'] = df_promo.groupby(['item_nbr', 'store_nbr', 'promo_break']).cumcount() + 1\ndf_promo.loc[~df_promo['onpromotion'], 'consec_promo_days'] = 0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:04.209709Z","iopub.execute_input":"2025-06-02T22:57:04.209969Z","iopub.status.idle":"2025-06-02T22:57:34.806317Z","shell.execute_reply.started":"2025-06-02T22:57:04.209948Z","shell.execute_reply":"2025-06-02T22:57:34.80557Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (10, 6))\nsns.histplot(df_promo[df_promo['consec_promo_days'] > 0]['consec_promo_days'], bins = 50, edgecolor = 'black')\nplt.xlabel('Consecutive Promotion Days')\nplt.ylabel('Frequency')\nplt.title('Distribution of Promotion Streak Lengths')\nplt.xticks(range(0, 300, 10))\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:34.807144Z","iopub.execute_input":"2025-06-02T22:57:34.807397Z","iopub.status.idle":"2025-06-02T22:57:39.857814Z","shell.execute_reply.started":"2025-06-02T22:57:34.807375Z","shell.execute_reply":"2025-06-02T22:57:39.857027Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"max_streaks = df_promo.groupby('item_nbr')['consec_promo_days'].max().sort_values(ascending = False).nlargest(10)\nitem_type = items_df[items_df['item_nbr'].isin(max_streaks.index)][['item_nbr', 'family']]\n\nitem_type['label'] = max_streaks.index.astype(str) + ' (' + item_type['family'] + ')'\nitem_type['max_streaks'] = max_streaks.values\nitem_type = item_type.sort_values('max_streaks', ascending = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:39.85855Z","iopub.execute_input":"2025-06-02T22:57:39.858751Z","iopub.status.idle":"2025-06-02T22:57:41.701831Z","shell.execute_reply.started":"2025-06-02T22:57:39.858736Z","shell.execute_reply":"2025-06-02T22:57:41.701239Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"item_type.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:41.702388Z","iopub.execute_input":"2025-06-02T22:57:41.702613Z","iopub.status.idle":"2025-06-02T22:57:41.709791Z","shell.execute_reply.started":"2025-06-02T22:57:41.702597Z","shell.execute_reply":"2025-06-02T22:57:41.709122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (15, 8))\nsns.barplot(y = item_type['label'], \n            x = item_type['max_streaks'], \n            hue = item_type['family'],\n            dodge = False,\n            alpha = 0.8, \n            width = 0.5, \n            orient = 'h',\n            edgecolor = 'black')\n\nplt.title('Top 10 Items by Longest Promotion Streak')\nplt.xlabel('Max Consecutive Promotion Days')\nplt.ylabel('Item Number')\nplt.yticks(fontsize = 10)\nplt.legend(fontsize = 10, loc = 'best')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:41.710462Z","iopub.execute_input":"2025-06-02T22:57:41.710751Z","iopub.status.idle":"2025-06-02T22:57:42.165452Z","shell.execute_reply.started":"2025-06-02T22:57:41.710735Z","shell.execute_reply":"2025-06-02T22:57:42.16483Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"store_sales = train_df.groupby('store_nbr', as_index = False)['unit_sales'].sum()\nstore_sales = store_sales.merge(stores_df, on = 'store_nbr')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:42.166241Z","iopub.execute_input":"2025-06-02T22:57:42.166449Z","iopub.status.idle":"2025-06-02T22:57:44.640231Z","shell.execute_reply.started":"2025-06-02T22:57:42.166426Z","shell.execute_reply":"2025-06-02T22:57:44.639715Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"store_sales = store_sales.groupby('type', as_index = False)['unit_sales'].sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:44.641005Z","iopub.execute_input":"2025-06-02T22:57:44.641501Z","iopub.status.idle":"2025-06-02T22:57:44.646163Z","shell.execute_reply.started":"2025-06-02T22:57:44.641478Z","shell.execute_reply":"2025-06-02T22:57:44.645366Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"store_sales.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:44.646934Z","iopub.execute_input":"2025-06-02T22:57:44.647205Z","iopub.status.idle":"2025-06-02T22:57:44.661738Z","shell.execute_reply.started":"2025-06-02T22:57:44.647184Z","shell.execute_reply":"2025-06-02T22:57:44.661089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (8, 8))\nstore_labels = ['Large Stores', 'Medium Stores', 'Small Stores', 'Convenient Stores', 'Special Stores']\ncolors = ['lightcoral', 'yellowgreen', 'gold', 'lightskyblue', 'lightcyan']\n\nexplode = [0.06, 0, 0, 0, 0.06]\n\nplt.pie(x = store_sales['unit_sales'],\n        labels = store_labels,\n        explode = explode,\n        autopct = '%0.2f%%',\n        colors = colors,\n        shadow = True,\n        startangle = 220,\n        wedgeprops = {'linewidth': 1, 'edgecolor': 'black'},\n        textprops = {'fontsize': 13})\n\nplt.title('Store Sales Distribution by Store Type', fontsize = 18)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:44.665098Z","iopub.execute_input":"2025-06-02T22:57:44.665341Z","iopub.status.idle":"2025-06-02T22:57:44.838358Z","shell.execute_reply.started":"2025-06-02T22:57:44.665325Z","shell.execute_reply":"2025-06-02T22:57:44.837694Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stores_df.head(), '', train_df.head()\n# Number of stores per city and state (plot on 1-axis)\n# Store Type by Cities (which type dominate in different cities) done","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:44.839168Z","iopub.execute_input":"2025-06-02T22:57:44.839413Z","iopub.status.idle":"2025-06-02T22:57:44.847554Z","shell.execute_reply.started":"2025-06-02T22:57:44.839392Z","shell.execute_reply":"2025-06-02T22:57:44.846883Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"city_store = stores_df.groupby(['city', 'state']).size().reset_index(name = 'store_count')\ncity_store.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:44.84841Z","iopub.execute_input":"2025-06-02T22:57:44.848677Z","iopub.status.idle":"2025-06-02T22:57:44.864122Z","shell.execute_reply.started":"2025-06-02T22:57:44.848661Z","shell.execute_reply":"2025-06-02T22:57:44.863589Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"city_stype = stores_df.groupby(['city', 'type']).size().reset_index(name = 'store_count')\npivoted_city_stype = city_stype.pivot(index = 'city', columns = 'type', values = 'store_count').fillna(0)\n# type A B C D E\n# city\n# Ambato 1 1 1 1 eg","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:44.864617Z","iopub.execute_input":"2025-06-02T22:57:44.864852Z","iopub.status.idle":"2025-06-02T22:57:44.886987Z","shell.execute_reply.started":"2025-06-02T22:57:44.864836Z","shell.execute_reply":"2025-06-02T22:57:44.886549Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pivoted_city_stype.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:44.887515Z","iopub.execute_input":"2025-06-02T22:57:44.887706Z","iopub.status.idle":"2025-06-02T22:57:44.896511Z","shell.execute_reply.started":"2025-06-02T22:57:44.887693Z","shell.execute_reply":"2025-06-02T22:57:44.895963Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pivoted_city_stype.plot(kind = 'bar', width = 1.3, figsize = (10, 6), edgecolor = 'black')\n\nplt.xlabel('City')\nplt.ylabel('Number of Stores')\nplt.title('Number of Stores by Type in Cities')\n\nplt.legend(title = 'Store Type', labels = store_labels)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:44.89705Z","iopub.execute_input":"2025-06-02T22:57:44.897246Z","iopub.status.idle":"2025-06-02T22:57:45.323926Z","shell.execute_reply.started":"2025-06-02T22:57:44.897232Z","shell.execute_reply":"2025-06-02T22:57:45.323144Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"trans_df.head()\n# What store had the highest burst of transactions in a span of date-time? What store had the lowest?","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:45.324847Z","iopub.execute_input":"2025-06-02T22:57:45.325116Z","iopub.status.idle":"2025-06-02T22:57:45.332592Z","shell.execute_reply.started":"2025-06-02T22:57:45.32509Z","shell.execute_reply":"2025-06-02T22:57:45.331916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"store_trans = trans_df.groupby(['store_nbr'], )['transactions'].sum().sort_values(ascending = False).reset_index()\nstore_trans = store_trans.merge(stores_df[['store_nbr', 'type']], on = 'store_nbr')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:45.333347Z","iopub.execute_input":"2025-06-02T22:57:45.333621Z","iopub.status.idle":"2025-06-02T22:57:45.353587Z","shell.execute_reply.started":"2025-06-02T22:57:45.333597Z","shell.execute_reply":"2025-06-02T22:57:45.353073Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"store_trans['type'] = store_trans['type'].map({'A' : 'Large Stores', 'B' : 'Medium Stores', 'C' : 'Small Stores', 'D' : 'Convenient Stores', 'E' : 'Special Stores'})\nstore_trans.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:45.354199Z","iopub.execute_input":"2025-06-02T22:57:45.354409Z","iopub.status.idle":"2025-06-02T22:57:45.361703Z","shell.execute_reply.started":"2025-06-02T22:57:45.354385Z","shell.execute_reply":"2025-06-02T22:57:45.361007Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig = px.treemap(\n    store_trans,\n    path = [px.Constant('Transactions by Store Size'), 'type', 'store_nbr'],\n    hover_data = ['store_nbr', 'type', 'transactions'],\n    values = 'transactions',\n    color = 'type',\n)\nfig.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:45.362403Z","iopub.execute_input":"2025-06-02T22:57:45.362657Z","iopub.status.idle":"2025-06-02T22:57:46.976485Z","shell.execute_reply.started":"2025-06-02T22:57:45.362633Z","shell.execute_reply":"2025-06-02T22:57:46.975867Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (10, 6))\n\ndaily_trans = trans_df.groupby('date')['transactions'].sum()\nrolling_30 = daily_trans.shift(1).rolling(30).mean()\n\nlowest_spikes = daily_trans.nsmallest(5)\n\ndaily_trans.plot(zorder = 1, alpha = 1, label = 'Transactions')\nrolling_30.plot(alpha = 0.9, color = 'red', label = '30 Days Rolling Average')\n\nplt.scatter(lowest_spikes.index, lowest_spikes.values + 1e2, marker = 'x', color = 'red', alpha = 1, zorder = 2)\n\nfor d, v in lowest_spikes.items():\n    plt.text(x = d, y = v - 7.5e3, s = str(f'{d.year}-{d.month}-{d.day}'), horizontalalignment = 'center', color = 'red')\n\nplt.xlabel('Date')\nplt.ylabel('Transactions')\nplt.title('Number of Transactions over Time')\nplt.legend()\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:46.977169Z","iopub.execute_input":"2025-06-02T22:57:46.977411Z","iopub.status.idle":"2025-06-02T22:57:47.245762Z","shell.execute_reply.started":"2025-06-02T22:57:46.977394Z","shell.execute_reply":"2025-06-02T22:57:47.245083Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"events_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:47.246623Z","iopub.execute_input":"2025-06-02T22:57:47.246887Z","iopub.status.idle":"2025-06-02T22:57:47.255345Z","shell.execute_reply.started":"2025-06-02T22:57:47.246864Z","shell.execute_reply":"2025-06-02T22:57:47.254771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"events_df['month'] = events_df['date'].dt.month","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:47.255942Z","iopub.execute_input":"2025-06-02T22:57:47.2561Z","iopub.status.idle":"2025-06-02T22:57:47.267034Z","shell.execute_reply.started":"2025-06-02T22:57:47.256087Z","shell.execute_reply":"2025-06-02T22:57:47.266509Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize = (9, 6))\n\nplt.subplot(2, 2, 1)\nsns.histplot(events_df['type'])\n\nplt.title('Distribution of Event Types')\nplt.xlabel('Event')\nplt.ylabel('Frequency')\nplt.xticks(fontsize = 8)\n\nplt.subplot(2, 2, 2)\ntype_locale = events_df.groupby(['type', 'locale']).size().reset_index(name = 'count')\ntype_locale = type_locale.pivot(index = 'locale', columns = 'type', values = 'count').fillna(0).reset_index()\ntype_locale = type_locale.melt(id_vars = 'locale', var_name = 'type', value_name = 'count') # Convert to long format using melt\n\nsns.barplot(x = type_locale['type'], y = type_locale['count'], hue = type_locale['locale'], edgecolor = 'black')\nplt.title('Events Locality')\nplt.xlabel('Event')\nplt.ylabel('Frequency')\nplt.xticks(fontsize = 8)\nplt.legend(fontsize = 8)\n\nplt.subplot(2, 2, 3)\ntype_transferred = events_df.groupby(['type', 'transferred']).size().reset_index(name = 'count')\ntype_transferred['transferred'] = type_transferred['transferred'].map({False : 'On time', True : 'Rescheduled'})\n\nsns.barplot(x = type_transferred['type'], y = type_transferred['count'], hue = type_transferred['transferred'], edgecolor = 'black')\nplt.title('Distribution of Rescheduled Events')\nplt.xlabel('Event')\nplt.ylabel('Frequency')\nplt.xticks(fontsize = 8)\nplt.legend(fontsize = 8)\n\nplt.subplot(2, 2, 4)\ntype_month = events_df.groupby(['type', 'month']).size().reset_index(name = 'count').sort_values('month')\ntype_month['month_name'] = pd.to_datetime(type_month['month'], format = '%m').dt.strftime('%b')\n\nsns.barplot(x = type_month['month_name'], y = type_month['count'], hue = type_month['type'], edgecolor = 'black')\n\nplt.title('Events Seasonality')\nplt.xlabel('Month')\nplt.ylabel('Frequency')\nplt.xticks(fontsize = 8)\nplt.legend(fontsize = 8, loc = 'upper left', ncol = 2, handlelength = 0.6, handleheight = 0.6)\n\nplt.suptitle('Events Analysis')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T22:57:47.267678Z","iopub.execute_input":"2025-06-02T22:57:47.267868Z","iopub.status.idle":"2025-06-02T22:57:48.135462Z","shell.execute_reply.started":"2025-06-02T22:57:47.267854Z","shell.execute_reply":"2025-06-02T22:57:48.134852Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Utilizing Dask for efficient data manipulation\n# Training set size > 1M samples\nimport dask.dataframe as dd","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:27:15.89475Z","iopub.execute_input":"2025-06-03T04:27:15.894971Z","iopub.status.idle":"2025-06-03T04:27:18.161228Z","shell.execute_reply.started":"2025-06-03T04:27:15.894954Z","shell.execute_reply":"2025-06-03T04:27:18.160694Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Read data as Dask dataframes\ntrain_dd = dd.read_csv('/kaggle/working/train.csv')\nstores_dd = dd.read_csv('/kaggle/working/stores.csv')\nitems_dd = dd.read_csv('/kaggle/working/items.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:27:18.161897Z","iopub.execute_input":"2025-06-03T04:27:18.162267Z","iopub.status.idle":"2025-06-03T04:27:18.22355Z","shell.execute_reply.started":"2025-06-03T04:27:18.16225Z","shell.execute_reply":"2025-06-03T04:27:18.222974Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dd = train_dd[['date', 'store_nbr', 'item_nbr', 'unit_sales']]\ntrain_dd.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:27:18.224243Z","iopub.execute_input":"2025-06-03T04:27:18.224473Z","iopub.status.idle":"2025-06-03T04:27:20.576672Z","shell.execute_reply.started":"2025-06-03T04:27:18.224455Z","shell.execute_reply":"2025-06-03T04:27:20.57582Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stores_dd.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T02:53:50.868141Z","iopub.execute_input":"2025-06-03T02:53:50.868494Z","iopub.status.idle":"2025-06-03T02:53:50.885768Z","shell.execute_reply.started":"2025-06-03T02:53:50.868469Z","shell.execute_reply":"2025-06-03T02:53:50.885166Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"items_dd.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T02:53:50.887607Z","iopub.execute_input":"2025-06-03T02:53:50.888038Z","iopub.status.idle":"2025-06-03T02:53:50.908774Z","shell.execute_reply.started":"2025-06-03T02:53:50.888022Z","shell.execute_reply":"2025-06-03T02:53:50.907995Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Parse date after bringing to memory so kernel doesnt crash.\ntrain_dd_pd = train_dd.compute() # Bring in memory","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:27:20.577859Z","iopub.execute_input":"2025-06-03T04:27:20.578206Z","iopub.status.idle":"2025-06-03T04:28:07.85211Z","shell.execute_reply.started":"2025-06-03T04:27:20.578171Z","shell.execute_reply":"2025-06-03T04:28:07.85154Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dd_pd['date'] = pd.to_datetime(train_dd_pd['date']) # Parse date","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:28:07.85285Z","iopub.execute_input":"2025-06-03T04:28:07.853099Z","iopub.status.idle":"2025-06-03T04:28:32.684627Z","shell.execute_reply.started":"2025-06-03T04:28:07.853073Z","shell.execute_reply":"2025-06-03T04:28:32.68381Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stores_dd = stores_dd.compute()\nitems_dd = items_dd.compute()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create LightFM dataset and find primary data\ndataset = Dataset()\ndataset.fit(train_dd_pd['store_nbr'], train_dd_pd['item_nbr']) # Fit user_id, item_id","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:29:55.176522Z","iopub.execute_input":"2025-06-03T04:29:55.177267Z","iopub.status.idle":"2025-06-03T04:30:56.009664Z","shell.execute_reply.started":"2025-06-03T04:29:55.177243Z","shell.execute_reply":"2025-06-03T04:30:56.009077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fit partial data that include user, items attributes\nuser_metadata_features = stores_dd['type'].apply(lambda x: f'type={x}').unique().tolist()\nitem_metadata_features = items_dd['family'].apply(lambda x: f'family={x}').unique().tolist()\n\n# Register these features with the dataset\ndataset.fit_partial(\n    users=train_dd_pd['store_nbr'],\n    items=train_dd_pd['item_nbr'],\n    user_features=user_metadata_features,\n    item_features=item_metadata_features\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:48:19.354965Z","iopub.execute_input":"2025-06-03T04:48:19.355508Z","iopub.status.idle":"2025-06-03T04:49:16.067241Z","shell.execute_reply.started":"2025-06-03T04:48:19.355487Z","shell.execute_reply":"2025-06-03T04:49:16.066468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import psutil\nprint(f'Current memory usage: {psutil.virtual_memory().percent}%')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:49:28.688187Z","iopub.execute_input":"2025-06-03T04:49:28.688922Z","iopub.status.idle":"2025-06-03T04:49:28.693086Z","shell.execute_reply.started":"2025-06-03T04:49:28.688897Z","shell.execute_reply":"2025-06-03T04:49:28.69239Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 1. Filter unit_sales > 0 early\nfiltered = train_dd_pd[train_dd_pd['unit_sales'] > 0]\n\n# 2. Drop duplicates (LightFM only needs one row per interaction)\nfiltered = filtered.drop_duplicates(subset=['store_nbr', 'item_nbr'])\n\n# 3. Convert to list faster\ninteraction_pairs = filtered[['store_nbr', 'item_nbr']].values.tolist()\n\n# 4. Build interactions (should now finish in seconds)\ninteractions, _ = dataset.build_interactions(interaction_pairs)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:49:31.224018Z","iopub.execute_input":"2025-06-03T04:49:31.224364Z","iopub.status.idle":"2025-06-03T04:49:46.071637Z","shell.execute_reply.started":"2025-06-03T04:49:31.22432Z","shell.execute_reply":"2025-06-03T04:49:46.070806Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Build features for items and users\n\nstores_dd['type_str'] = 'type=' + stores_dd['type']\nitems_dd['family_str'] = 'family=' + items_dd['family']\n\n# Merge back to train\ntrain_meta = train_dd_pd.merge(stores_dd[['store_nbr', 'type_str']], on='store_nbr')\ntrain_meta = train_meta.merge(items_dd[['item_nbr', 'family_str']], on='item_nbr')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:49:49.103947Z","iopub.execute_input":"2025-06-03T04:49:49.104211Z","iopub.status.idle":"2025-06-03T04:50:18.015933Z","shell.execute_reply.started":"2025-06-03T04:49:49.104191Z","shell.execute_reply":"2025-06-03T04:50:18.015368Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 1: Drop duplicates — we only want one row per store\nunique_user_features = train_meta[['store_nbr', 'type_str']].drop_duplicates()\n\n# Step 2: Convert to the format LightFM expects\nuser_feature_tuples = list(zip(unique_user_features['store_nbr'], unique_user_features['type_str'].apply(lambda x: [x])))\n\n# Step 3: Build features\nuser_features = dataset.build_user_features(user_feature_tuples)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:50:20.804512Z","iopub.execute_input":"2025-06-03T04:50:20.804758Z","iopub.status.idle":"2025-06-03T04:50:27.819088Z","shell.execute_reply.started":"2025-06-03T04:50:20.804743Z","shell.execute_reply":"2025-06-03T04:50:27.818277Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 1: Drop duplicate item-feature pairs\nunique_item_features = train_meta[['item_nbr', 'family_str']].drop_duplicates()\n\n# Step 2: Format for LightFM\nitem_feature_tuples = list(zip(\n    unique_item_features['item_nbr'],\n    unique_item_features['family_str'].apply(lambda x: [x])\n))\n\n# Step 3: Build LightFM item features\nitem_features = dataset.build_item_features(item_feature_tuples)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:50:31.610718Z","iopub.execute_input":"2025-06-03T04:50:31.61097Z","iopub.status.idle":"2025-06-03T04:50:40.292924Z","shell.execute_reply.started":"2025-06-03T04:50:31.61095Z","shell.execute_reply":"2025-06-03T04:50:40.292379Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = LightFM(loss='warp')  # good for implicit feedback","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:50:47.911084Z","iopub.execute_input":"2025-06-03T04:50:47.911766Z","iopub.status.idle":"2025-06-03T04:50:47.915461Z","shell.execute_reply.started":"2025-06-03T04:50:47.911742Z","shell.execute_reply":"2025-06-03T04:50:47.914808Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.fit(interactions, user_features=user_features, item_features=item_features, epochs=10, num_threads=4)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:50:49.532822Z","iopub.execute_input":"2025-06-03T04:50:49.533035Z","iopub.status.idle":"2025-06-03T04:50:50.605729Z","shell.execute_reply.started":"2025-06-03T04:50:49.533021Z","shell.execute_reply":"2025-06-03T04:50:50.604714Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"user_id = 0  # internal user index\nscores = model.predict(user_id, np.arange(interactions.shape[1]), user_features=user_features, item_features=item_features)\ntop_items = np.argsort(-scores)[:10]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:51:01.680954Z","iopub.execute_input":"2025-06-03T04:51:01.681457Z","iopub.status.idle":"2025-06-03T04:51:01.6868Z","shell.execute_reply.started":"2025-06-03T04:51:01.681434Z","shell.execute_reply":"2025-06-03T04:51:01.6862Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Top 10 recommended item indices:\", top_items)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:51:37.086864Z","iopub.execute_input":"2025-06-03T04:51:37.087132Z","iopub.status.idle":"2025-06-03T04:51:37.091699Z","shell.execute_reply.started":"2025-06-03T04:51:37.087112Z","shell.execute_reply":"2025-06-03T04:51:37.090924Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from lightfm.evaluation import precision_at_k, recall_at_k\n\nprec = precision_at_k(model, interactions, k=1000, user_features=user_features, item_features=item_features).mean()\nrec = recall_at_k(model, interactions, k=1000, user_features=user_features, item_features=item_features).mean()\n\nprint(f'Precision@100: {prec:.4f}')\nprint(f'Recall@100:    {rec:.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:57:55.713465Z","iopub.execute_input":"2025-06-03T04:57:55.714191Z","iopub.status.idle":"2025-06-03T04:57:59.645202Z","shell.execute_reply.started":"2025-06-03T04:57:55.714169Z","shell.execute_reply":"2025-06-03T04:57:59.644619Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def recommend_items_for_store(store_nbr, model, dataset, k, user_features, item_features):\n    store_nbr_to_id = dataset.mapping()[0]\n    id_to_item_nbr = {v: k for k, v in dataset.mapping()[2].items()}\n    \n    if store_nbr not in store_nbr_to_id:\n        raise ValueError(f\"Store {store_nbr} not in training data.\")\n\n    user_id = store_nbr_to_id[store_nbr]\n    n_items = len(id_to_item_nbr)\n\n    scores = model.predict(user_id, np.arange(n_items), user_features=user_features, item_features=item_features)\n    top_item_ids = np.argsort(-scores)[:k]\n    top_real_items = [id_to_item_nbr[i] for i in top_item_ids]\n    \n    return top_real_items","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:55:52.580405Z","iopub.execute_input":"2025-06-03T04:55:52.580657Z","iopub.status.idle":"2025-06-03T04:55:52.585871Z","shell.execute_reply.started":"2025-06-03T04:55:52.58064Z","shell.execute_reply":"2025-06-03T04:55:52.585085Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"recommendations = recommend_items_for_store(\n    store_nbr=25,\n    model=model,\n    dataset=dataset,\n    k=10,\n    user_features=user_features,\n    item_features=item_features\n)\n\nprint(\"Top recommended item_nbrs for store 25:\", recommendations)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-03T04:53:01.52218Z","iopub.execute_input":"2025-06-03T04:53:01.522798Z","iopub.status.idle":"2025-06-03T04:53:01.528158Z","shell.execute_reply.started":"2025-06-03T04:53:01.522778Z","shell.execute_reply":"2025-06-03T04:53:01.527503Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"--------------------------","metadata":{}}]}