{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":7391,"databundleVersionId":44328,"sourceType":"competition"},{"sourceId":84631216,"sourceType":"kernelVersion"}],"dockerImageVersionId":31013,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"raw","source":"PRACTICA 1\nEl siguiente codigo usa las librerias estudiadas y tamnbien se prueba nuevas librerias que se han investigado y probado para una preddcion de ventas de comestibles.\n","metadata":{}},{"cell_type":"code","source":"\n\nimport numpy as np\nimport pandas as pd\n\n\n\nimport os\nfor dirname, _, filenames in os.walk('D:\\favorita'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2025-01-30T02:38:07.681399Z","iopub.execute_input":"2025-01-30T02:38:07.681744Z","iopub.status.idle":"2025-01-30T02:38:07.695797Z","shell.execute_reply.started":"2025-01-30T02:38:07.681656Z","shell.execute_reply":"2025-01-30T02:38:07.695199Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Explore and prepare the data","metadata":{}},{"cell_type":"code","source":"items = pd.read_csv(\"D:\\favorita/items.csv\")\nholiday_events = pd.read_csv(\"D:\\favorita/holidays_events.csv\", parse_dates=['date'])\nstores = pd.read_csv(\"D:\\favorita/stores.csv\")\noil = pd.read_csv(\"D:\\favorita/oil.csv\", parse_dates=['date'])\ntransactions = pd.read_csv(\"D:\\favorita/transactions.csv\", parse_dates=['date'])\ntrain = pd.read_csv('D:\\favorita/train.csv', parse_dates = ['date'])\n","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:39:40.195115Z","iopub.execute_input":"2025-01-30T02:39:40.195302Z","iopub.status.idle":"2025-01-30T02:40:53.312780Z","shell.execute_reply.started":"2025-01-30T02:39:40.195281Z","shell.execute_reply":"2025-01-30T02:40:53.312142Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:40:53.313828Z","iopub.execute_input":"2025-01-30T02:40:53.314048Z","iopub.status.idle":"2025-01-30T02:40:53.330228Z","shell.execute_reply.started":"2025-01-30T02:40:53.314021Z","shell.execute_reply":"2025-01-30T02:40:53.329474Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.tail()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:40:53.331099Z","iopub.execute_input":"2025-01-30T02:40:53.331285Z","iopub.status.idle":"2025-01-30T02:40:53.341356Z","shell.execute_reply.started":"2025-01-30T02:40:53.331262Z","shell.execute_reply":"2025-01-30T02:40:53.340712Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Nulls in Oil columns: {0} => {1}\".format(oil.columns.values,oil.isnull().any().values))\nprint(\"=\"*70)\nprint(\"Nulls in holiday_events columns: {0} => {1}\".format(holiday_events.columns.values,holiday_events.isnull().any().values))\nprint(\"=\"*70)\nprint(\"Nulls in stores columns: {0} => {1}\".format(stores.columns.values,stores.isnull().any().values))\nprint(\"=\"*70)\nprint(\"Nulls in transactions columns: {0} => {1}\".format(transactions.columns.values,transactions.isnull().any().values))\nprint(\"=\"*70)\nprint(\"Nulls in train columns: {0} => {1}\".format(train.columns.values,train.isnull().any().values))\n","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:40:53.342372Z","iopub.execute_input":"2025-01-30T02:40:53.342579Z","iopub.status.idle":"2025-01-30T02:40:59.600763Z","shell.execute_reply.started":"2025-01-30T02:40:53.342552Z","shell.execute_reply":"2025-01-30T02:40:59.600212Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"oil['dcoilwtico'] = oil['dcoilwtico'].interpolate()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:40:59.602593Z","iopub.execute_input":"2025-01-30T02:40:59.602798Z","iopub.status.idle":"2025-01-30T02:40:59.609667Z","shell.execute_reply.started":"2025-01-30T02:40:59.602768Z","shell.execute_reply":"2025-01-30T02:40:59.609150Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"oil['dcoilwtico'] = oil['dcoilwtico'].fillna(method='bfill')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:40:59.610404Z","iopub.execute_input":"2025-01-30T02:40:59.610565Z","iopub.status.idle":"2025-01-30T02:40:59.624715Z","shell.execute_reply.started":"2025-01-30T02:40:59.610545Z","shell.execute_reply":"2025-01-30T02:40:59.624170Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"oil.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:40:59.625489Z","iopub.execute_input":"2025-01-30T02:40:59.625671Z","iopub.status.idle":"2025-01-30T02:40:59.635288Z","shell.execute_reply.started":"2025-01-30T02:40:59.625649Z","shell.execute_reply":"2025-01-30T02:40:59.634782Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:40:59.636166Z","iopub.execute_input":"2025-01-30T02:40:59.636347Z","iopub.status.idle":"2025-01-30T02:41:06.487820Z","shell.execute_reply.started":"2025-01-30T02:40:59.636325Z","shell.execute_reply":"2025-01-30T02:41:06.487314Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['onpromotion'] = train['onpromotion'].fillna('False')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:41:06.488642Z","iopub.execute_input":"2025-01-30T02:41:06.489094Z","iopub.status.idle":"2025-01-30T02:41:17.394639Z","shell.execute_reply.started":"2025-01-30T02:41:06.489066Z","shell.execute_reply":"2025-01-30T02:41:17.394071Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:41:17.395599Z","iopub.execute_input":"2025-01-30T02:41:17.395835Z","iopub.status.idle":"2025-01-30T02:41:24.543351Z","shell.execute_reply.started":"2025-01-30T02:41:17.395802Z","shell.execute_reply":"2025-01-30T02:41:24.542787Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"sales_oil = train.groupby(\"date\")[\"unit_sales\"].sum().reset_index()\nsales_oil = sales_oil.merge(oil, on=\"date\", how=\"left\") \n\nplt.figure(figsize=(14, 6))\n\n\nplt.plot(sales_oil[\"date\"], sales_oil[\"unit_sales\"], label=\"Total Unit Sales\", color=\"blue\", alpha=0.6)\n\n\nplt.twinx()\nplt.plot(sales_oil[\"date\"], sales_oil[\"dcoilwtico\"], label=\"Oil Prices\", color=\"red\", alpha=0.6)\n\nplt.title(\"Daily Sales vs Oil Prices\")\nplt.xlabel(\"Date\")\nplt.legend([\"Unit Sales\", \"Oil Prices\"])\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:41:24.544274Z","iopub.execute_input":"2025-01-30T02:41:24.544463Z","iopub.status.idle":"2025-01-30T02:41:27.301508Z","shell.execute_reply.started":"2025-01-30T02:41:24.544440Z","shell.execute_reply":"2025-01-30T02:41:27.301022Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"correlation = sales_oil[\"unit_sales\"].corr(sales_oil[\"dcoilwtico\"])\nprint(f\"Correlation between oil price and sales: {correlation:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:41:27.302366Z","iopub.execute_input":"2025-01-30T02:41:27.302577Z","iopub.status.idle":"2025-01-30T02:41:27.307179Z","shell.execute_reply.started":"2025-01-30T02:41:27.302552Z","shell.execute_reply":"2025-01-30T02:41:27.306524Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nsales_oil[\"oil_lag_7\"] = sales_oil[\"dcoilwtico\"].shift(7)  # Lag by 7 days\nsales_oil[\"oil_lag_3\"] = sales_oil[\"dcoilwtico\"].shift(3)  # Lag by 3 days\n\n\ncorrelation_lag_7 = sales_oil[\"unit_sales\"].corr(sales_oil[\"oil_lag_7\"])\ncorrelation_lag_3 = sales_oil[\"unit_sales\"].corr(sales_oil[\"oil_lag_3\"])\n\nprint(f\"Correlation with 7-day lag: {correlation_lag_7:.4f}\")\nprint(f\"Correlation with 3-day lag: {correlation_lag_3:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:41:27.308140Z","iopub.execute_input":"2025-01-30T02:41:27.308356Z","iopub.status.idle":"2025-01-30T02:41:27.320882Z","shell.execute_reply.started":"2025-01-30T02:41:27.308330Z","shell.execute_reply":"2025-01-30T02:41:27.320298Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"store_sales = train.groupby(\"store_nbr\")[\"unit_sales\"].sum().reset_index()\n\n\nstore_sales = store_sales.sort_values(by=\"unit_sales\", ascending=False)\n\nprint(store_sales.head())\n\nplt.figure(figsize=(12, 6))\nplt.bar(store_sales[\"store_nbr\"], store_sales[\"unit_sales\"], color=\"skyblue\")\nplt.title(\"Total Sales Per Store\")\nplt.xlabel(\"Store Number\")\nplt.ylabel(\"Total Sales\")\nplt.xticks(rotation=90)  # Rotate labels for better visibility\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:41:27.321717Z","iopub.execute_input":"2025-01-30T02:41:27.321955Z","iopub.status.idle":"2025-01-30T02:41:29.894339Z","shell.execute_reply.started":"2025-01-30T02:41:27.321924Z","shell.execute_reply":"2025-01-30T02:41:29.893800Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[\"year\"] = train[\"date\"].dt.year\ntrain[\"month\"] = train[\"date\"].dt.month\n\nmonthly_sales_by_year = train.groupby([\"year\", \"month\"])[\"unit_sales\"].sum().reset_index()\n\nimport seaborn as sns\n\nplt.figure(figsize=(12, 6))\nsns.lineplot(data=monthly_sales_by_year, x=\"month\", y=\"unit_sales\", hue=\"year\", palette=\"tab10\")\nplt.title(\"Monthly Sales Trend Across Years\")\nplt.xlabel(\"Month\")\nplt.ylabel(\"Total Sales\")\nplt.xticks(range(1, 13))\nplt.legend(title=\"Year\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:41:29.895174Z","iopub.execute_input":"2025-01-30T02:41:29.895350Z","iopub.status.idle":"2025-01-30T02:41:57.212367Z","shell.execute_reply.started":"2025-01-30T02:41:29.895329Z","shell.execute_reply":"2025-01-30T02:41:57.211798Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntrain[\"year_month\"] = train[\"date\"].dt.to_period(\"M\")  # Format: YYYY-MM\n\n\nmonthly_sales = train.groupby(\"year_month\")[\"unit_sales\"].sum().reset_index()\n\n\nmonthly_sales[\"year_month\"] = monthly_sales[\"year_month\"].astype(str)\n\nplt.figure(figsize=(14, 6))\nsns.lineplot(data=monthly_sales, x=\"year_month\", y=\"unit_sales\", marker=\"o\", linestyle=\"-\", color=\"b\")\nplt.title(\"Consecutive Monthly Sales Trend Over Years\")\nplt.xlabel(\"Year-Month\")\nplt.ylabel(\"Total Sales\")\nplt.xticks(rotation=45)  # Rotate labels for better readability\nplt.grid(True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:41:57.213267Z","iopub.execute_input":"2025-01-30T02:41:57.213458Z","iopub.status.idle":"2025-01-30T02:42:11.619889Z","shell.execute_reply.started":"2025-01-30T02:41:57.213435Z","shell.execute_reply":"2025-01-30T02:42:11.619323Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_with_clusters = pd.merge(train, stores[['store_nbr', 'cluster']], on='store_nbr', how='left')\ntotal_sales_by_cluster = train_with_clusters.groupby('cluster')['unit_sales'].sum().reset_index()\n\nplt.figure(figsize=(8, 6))\nsns.barplot(x='cluster', y='unit_sales', data=total_sales_by_cluster, palette='viridis')\nplt.title('Total Sales by Store Cluster')\nplt.xlabel('Cluster')\nplt.ylabel('Total Sales')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:42:11.620808Z","iopub.execute_input":"2025-01-30T02:42:11.620997Z","iopub.status.idle":"2025-01-30T02:42:36.765360Z","shell.execute_reply.started":"2025-01-30T02:42:11.620974Z","shell.execute_reply":"2025-01-30T02:42:36.764759Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"daily_sales = train.groupby('date')['unit_sales'].sum().reset_index()\ndaily_sales['rolling_avg'] = daily_sales['unit_sales'].rolling(window=30).mean()\n\n\nimport matplotlib.pyplot as plt\n\nplt.figure(figsize=(12, 6))\nplt.plot(daily_sales['date'], daily_sales['unit_sales'], label='Daily Sales', alpha=0.5)\nplt.plot(daily_sales['date'], daily_sales['rolling_avg'], label='30-Day Rolling Avg', color='red')\nplt.title('Sales Trend Over Time')\nplt.xlabel('Date')\nplt.ylabel('Sales')\nplt.legend()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:42:36.766248Z","iopub.execute_input":"2025-01-30T02:42:36.766416Z","iopub.status.idle":"2025-01-30T02:42:39.413436Z","shell.execute_reply.started":"2025-01-30T02:42:36.766395Z","shell.execute_reply":"2025-01-30T02:42:39.412924Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"daily_sales['month'] = daily_sales['date'].dt.month\n\n\nmonthly_sales = daily_sales.groupby('month')['unit_sales'].mean().reset_index()\n\n\nplt.figure(figsize=(10, 6))\nplt.plot(monthly_sales['month'], monthly_sales['unit_sales'], marker='o')\nplt.title('Monthly Seasonality in Sales')\nplt.xlabel('Month')\nplt.ylabel('Average Sales')\nplt.xticks(range(1, 13), ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'])\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:42:39.414309Z","iopub.execute_input":"2025-01-30T02:42:39.414507Z","iopub.status.idle":"2025-01-30T02:42:39.543708Z","shell.execute_reply.started":"2025-01-30T02:42:39.414482Z","shell.execute_reply":"2025-01-30T02:42:39.543241Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ndaily_sales['day_of_week'] = daily_sales['date'].dt.dayofweek\n\n\nweekly_sales = daily_sales.groupby('day_of_week')['unit_sales'].mean().reset_index()\n\n\nplt.figure(figsize=(10, 6))\nplt.plot(weekly_sales['day_of_week'], weekly_sales['unit_sales'], marker='o')\nplt.title('Weekly Seasonality in Sales')\nplt.xlabel('Day of Week')\nplt.ylabel('Average Sales')\nplt.xticks(range(7), ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'])\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:42:39.547021Z","iopub.execute_input":"2025-01-30T02:42:39.547189Z","iopub.status.idle":"2025-01-30T02:42:39.665906Z","shell.execute_reply.started":"2025-01-30T02:42:39.547168Z","shell.execute_reply":"2025-01-30T02:42:39.665373Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from statsmodels.tsa.seasonal import seasonal_decompose\n\n\ndecomposition = seasonal_decompose(daily_sales.set_index('date')['unit_sales'], model='additive', period=365)\n\n\nplt.figure(figsize=(12, 8))\ndecomposition.plot()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:42:39.666652Z","iopub.execute_input":"2025-01-30T02:42:39.666980Z","iopub.status.idle":"2025-01-30T02:42:40.341988Z","shell.execute_reply.started":"2025-01-30T02:42:39.666956Z","shell.execute_reply":"2025-01-30T02:42:40.341484Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"Time related Features","metadata":{}},{"cell_type":"code","source":"\ntrain['day_of_week'] = train['date'].dt.dayofweek  # Monday=0, Sunday=6\ntrain['month'] = train['date'].dt.month\ntrain['year'] = train['date'].dt.year\ntrain['week_of_year'] = train['date'].dt.isocalendar().week\n\ntrain['is_weekend'] = train['day_of_week'].isin([5, 6]).astype(int)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:42:40.342798Z","iopub.execute_input":"2025-01-30T02:42:40.342967Z","iopub.status.idle":"2025-01-30T02:43:53.223428Z","shell.execute_reply.started":"2025-01-30T02:42:40.342946Z","shell.execute_reply":"2025-01-30T02:43:53.222662Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Lag Features","metadata":{}},{"cell_type":"code","source":"train['sales_lag_7'] = train.groupby(['store_nbr', 'item_nbr'])['unit_sales'].shift(7)\n\noil['oil_price_lag_7'] = oil['dcoilwtico'].shift(7)\n\ntransactions['transactions_lag_7'] = transactions['transactions'].shift(7)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:43:53.224499Z","iopub.execute_input":"2025-01-30T02:43:53.224699Z","iopub.status.idle":"2025-01-30T02:44:04.598879Z","shell.execute_reply.started":"2025-01-30T02:43:53.224674Z","shell.execute_reply":"2025-01-30T02:44:04.598277Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['sales_rolling_avg_7'] = train.groupby(['store_nbr', 'item_nbr'])['unit_sales'].transform(lambda x: x.rolling(window=7).mean())\n\noil['oil_price_rolling_avg_7'] = oil['dcoilwtico'].rolling(window=7).mean()\n\ntransactions['transactions_rolling_avg_7'] = transactions['transactions'].rolling(window=7).mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:44:04.599798Z","iopub.execute_input":"2025-01-30T02:44:04.599982Z","iopub.status.idle":"2025-01-30T02:45:47.457666Z","shell.execute_reply.started":"2025-01-30T02:44:04.599960Z","shell.execute_reply":"2025-01-30T02:45:47.457048Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"holiday_events['is_holiday'] = holiday_events['type'].apply(lambda x: 1 if x in ['Holiday', 'Event'] else 0)\n\ntrain = pd.merge(train, holiday_events[['date', 'is_holiday']], on='date', how='left')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:45:47.458644Z","iopub.execute_input":"2025-01-30T02:45:47.458853Z","iopub.status.idle":"2025-01-30T02:46:16.921670Z","shell.execute_reply.started":"2025-01-30T02:45:47.458828Z","shell.execute_reply":"2025-01-30T02:46:16.921058Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del holiday_events","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:46:16.922608Z","iopub.execute_input":"2025-01-30T02:46:16.922811Z","iopub.status.idle":"2025-01-30T02:46:16.925805Z","shell.execute_reply.started":"2025-01-30T02:46:16.922778Z","shell.execute_reply":"2025-01-30T02:46:16.925312Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['is_holiday'] = train['is_holiday'].fillna(0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:46:16.926686Z","iopub.execute_input":"2025-01-30T02:46:16.926904Z","iopub.status.idle":"2025-01-30T02:46:18.089105Z","shell.execute_reply.started":"2025-01-30T02:46:16.926881Z","shell.execute_reply":"2025-01-30T02:46:18.088495Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.merge(train, stores[['store_nbr', 'state', 'cluster']], on='store_nbr', how='left')\n\ntrain = pd.merge(train, items[['item_nbr', 'family', 'perishable']], on='item_nbr', how='left')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:46:18.089974Z","iopub.execute_input":"2025-01-30T02:46:18.090147Z","iopub.status.idle":"2025-01-30T02:47:31.728345Z","shell.execute_reply.started":"2025-01-30T02:46:18.090126Z","shell.execute_reply":"2025-01-30T02:47:31.727701Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del stores","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:47:31.729350Z","iopub.execute_input":"2025-01-30T02:47:31.729538Z","iopub.status.idle":"2025-01-30T02:47:31.732550Z","shell.execute_reply.started":"2025-01-30T02:47:31.729515Z","shell.execute_reply":"2025-01-30T02:47:31.731933Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del items\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:47:31.733427Z","iopub.execute_input":"2025-01-30T02:47:31.733626Z","iopub.status.idle":"2025-01-30T02:47:31.745975Z","shell.execute_reply.started":"2025-01-30T02:47:31.733603Z","shell.execute_reply":"2025-01-30T02:47:31.745391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gc\n\ngc.collect()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:47:31.746688Z","iopub.execute_input":"2025-01-30T02:47:31.746885Z","iopub.status.idle":"2025-01-30T02:47:31.863076Z","shell.execute_reply.started":"2025-01-30T02:47:31.746864Z","shell.execute_reply":"2025-01-30T02:47:31.862515Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.merge(train, oil[['date', 'dcoilwtico','oil_price_lag_7']], on='date', how='left')\n# train = pd.merge(train, transactions[['date','transactions']], on='date', how='left')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:47:31.863894Z","iopub.execute_input":"2025-01-30T02:47:31.864078Z","iopub.status.idle":"2025-01-30T02:48:10.521330Z","shell.execute_reply.started":"2025-01-30T02:47:31.864055Z","shell.execute_reply":"2025-01-30T02:48:10.520669Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del oil","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:48:10.522330Z","iopub.execute_input":"2025-01-30T02:48:10.522512Z","iopub.status.idle":"2025-01-30T02:48:10.525397Z","shell.execute_reply.started":"2025-01-30T02:48:10.522491Z","shell.execute_reply":"2025-01-30T02:48:10.524945Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"del transactions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:48:10.526175Z","iopub.execute_input":"2025-01-30T02:48:10.526343Z","iopub.status.idle":"2025-01-30T02:48:10.542276Z","shell.execute_reply.started":"2025-01-30T02:48:10.526322Z","shell.execute_reply":"2025-01-30T02:48:10.541786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:48:10.543055Z","iopub.execute_input":"2025-01-30T02:48:10.543224Z","iopub.status.idle":"2025-01-30T02:48:10.655918Z","shell.execute_reply.started":"2025-01-30T02:48:10.543202Z","shell.execute_reply":"2025-01-30T02:48:10.655405Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.iloc[7:].reset_index(drop=True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:48:10.656672Z","iopub.execute_input":"2025-01-30T02:48:10.656870Z","iopub.status.idle":"2025-01-30T02:48:46.214227Z","shell.execute_reply.started":"2025-01-30T02:48:10.656849Z","shell.execute_reply":"2025-01-30T02:48:46.213578Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:48:46.215141Z","iopub.execute_input":"2025-01-30T02:48:46.215323Z","iopub.status.idle":"2025-01-30T02:48:46.239465Z","shell.execute_reply.started":"2025-01-30T02:48:46.215301Z","shell.execute_reply":"2025-01-30T02:48:46.238871Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:48:46.240307Z","iopub.execute_input":"2025-01-30T02:48:46.240497Z","iopub.status.idle":"2025-01-30T02:49:07.444663Z","shell.execute_reply.started":"2025-01-30T02:48:46.240473Z","shell.execute_reply":"2025-01-30T02:49:07.444084Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntrain = train.drop(columns=['id', 'year_month'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:49:07.445570Z","iopub.execute_input":"2025-01-30T02:49:07.445746Z","iopub.status.idle":"2025-01-30T02:49:15.938610Z","shell.execute_reply.started":"2025-01-30T02:49:07.445725Z","shell.execute_reply":"2025-01-30T02:49:15.937942Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntrain['dcoilwtico'] = train['dcoilwtico'].fillna(method='ffill')\ntrain['oil_price_lag_7'] = train['oil_price_lag_7'].fillna(method='bfill')\n\n# Fill missing values with the mean of the column\ntrain['sales_lag_7'] = train['sales_lag_7'].fillna(train['sales_lag_7'].mean())\ntrain['sales_rolling_avg_7'] = train['sales_rolling_avg_7'].fillna(train['sales_rolling_avg_7'].mean())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:50:40.908648Z","iopub.execute_input":"2025-01-30T02:50:40.909509Z","iopub.status.idle":"2025-01-30T02:50:45.949305Z","shell.execute_reply.started":"2025-01-30T02:50:40.909472Z","shell.execute_reply":"2025-01-30T02:50:45.948561Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:50:45.950596Z","iopub.execute_input":"2025-01-30T02:50:45.950827Z","iopub.status.idle":"2025-01-30T02:51:07.146157Z","shell.execute_reply.started":"2025-01-30T02:50:45.950800Z","shell.execute_reply":"2025-01-30T02:51:07.145543Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\ncat_features  = ['store_nbr', 'item_nbr', 'state', 'cluster', 'family', 'onpromotion', 'weekofyear']\nlabel_encoders = {}\n\nfor col in cat_features:\n    label_encoders[col] = LabelEncoder()\n    train[col] = label_encoders[col].fit_transform(train[col])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:51:11.539456Z","iopub.execute_input":"2025-01-30T02:51:11.540283Z","iopub.status.idle":"2025-01-30T02:52:25.325731Z","shell.execute_reply.started":"2025-01-30T02:51:11.540246Z","shell.execute_reply":"2025-01-30T02:52:25.324740Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\nnum_features = [\"unit_sales\", \"dcoilwtico\", \"sales_rolling_avg_7\", \"oil_price_lag_7\"]\nscaler = StandardScaler()\ntrain[num_features] = scaler.fit_transform(train[num_features])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:52:25.327086Z","iopub.execute_input":"2025-01-30T02:52:25.327285Z","iopub.status.idle":"2025-01-30T02:52:46.704304Z","shell.execute_reply.started":"2025-01-30T02:52:25.327261Z","shell.execute_reply":"2025-01-30T02:52:46.703564Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['onpromotion'] = train['onpromotion'].astype(bool)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:57:11.650274Z","iopub.execute_input":"2025-01-30T02:57:11.650729Z","iopub.status.idle":"2025-01-30T02:57:13.638118Z","shell.execute_reply.started":"2025-01-30T02:57:11.650682Z","shell.execute_reply":"2025-01-30T02:57:13.637326Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['week_of_year'] = train['week_of_year'].astype(int)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:57:46.071569Z","iopub.execute_input":"2025-01-30T02:57:46.072539Z","iopub.status.idle":"2025-01-30T02:57:46.766179Z","shell.execute_reply.started":"2025-01-30T02:57:46.072503Z","shell.execute_reply":"2025-01-30T02:57:46.765424Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training Data","metadata":{}},{"cell_type":"code","source":"\nfeatures = [col for col in train.columns if col not in ['unit_sales', 'date']]\nX = train[features]\ny = train['unit_sales']\n\nprint(\"Features:\")\nprint(X.head())\n\nprint(\"\\nTarget Variable:\")\nprint(y.head())","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:57:51.579012Z","iopub.execute_input":"2025-01-30T02:57:51.579881Z","iopub.status.idle":"2025-01-30T02:58:05.585113Z","shell.execute_reply.started":"2025-01-30T02:57:51.579835Z","shell.execute_reply":"2025-01-30T02:58:05.584475Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, shuffle=False)\n\nprint(f\"Training data shape: {X_train.shape}, {y_train.shape}\")\nprint(f\"Validation data shape: {X_val.shape}, {y_val.shape}\")","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:58:05.586322Z","iopub.execute_input":"2025-01-30T02:58:05.586511Z","iopub.status.idle":"2025-01-30T02:58:14.681094Z","shell.execute_reply.started":"2025-01-30T02:58:05.586488Z","shell.execute_reply":"2025-01-30T02:58:14.680469Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from xgboost import XGBRegressor\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error\n\n\nmodel = XGBRegressor(n_estimators=100, learning_rate=0.1, random_state=42)\n\n\nmodel.fit(X_train, y_train)\n\n\ny_pred = model.predict(X_val)\n\n# Evaluate the model\nmse = mean_squared_error(y_val, y_pred)\nmae = mean_absolute_error(y_val, y_pred)\nprint(f\"Mean Squared Error (MSE): {mse}\")\nprint(f\"Mean Absolute Error (MAE): {mae}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:58:14.682085Z","iopub.execute_input":"2025-01-30T02:58:14.682282Z","execution_failed":"2025-01-30T04:39:22.770Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nxgb.plot_importance(model)\nplt.show()","metadata":{"trusted":true,"execution":{"execution_failed":"2025-01-30T04:39:22.771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.preprocessing import MinMaxScaler\n\n\nscaler = MinMaxScaler()\nX_scaled = scaler.fit_transform(X)\ny_scaled = scaler.fit_transform(y.values.reshape(-1, 1))\n\n\ndef create_sequences(data, timesteps=7):\n    X_seq, y_seq = [], []\n    for i in range(timesteps, len(data)):\n        X_seq.append(data[i-timesteps:i])\n        y_seq.append(data[i])\n    return np.array(X_seq), np.array(y_seq)\n\n\ntimesteps = 7\nX_seq, y_seq = create_sequences(X_scaled, timesteps)\n\n\nX_train_lstm, X_val_lstm, y_train_lstm, y_val_lstm = train_test_split(X_seq, y_seq, test_size=0.2, shuffle=False)\n\n\nprint(f\"LSTM Training data shape: {X_train_lstm.shape}, {y_train_lstm.shape}\")\nprint(f\"LSTM Validation data shape: {X_val_lstm.shape}, {y_val_lstm.shape}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import LSTM, Dense, Dropout\n\n\nlstm_model = Sequential([\n    LSTM(50, return_sequences=True, input_shape=(X_train_lstm.shape[1], X_train_lstm.shape[2])),\n    Dropout(0.2),\n    LSTM(50, return_sequences=False),\n    Dropout(0.2),\n    Dense(1)\n])\n\n\nlstm_model.compile(optimizer='adam', loss='mean_squared_error')\n\n# Train the model\nhistory = lstm_model.fit(\n    X_train_lstm, y_train_lstm,\n    validation_data=(X_val_lstm, y_val_lstm),\n    epochs=20,\n    batch_size=32,\n    verbose=1\n)\n\n\nplt.figure(figsize=(10, 6))\nplt.plot(history.history['loss'], label='Training Loss')\nplt.plot(history.history['val_loss'], label='Validation Loss')\nplt.title('LSTM - Training and Validation Loss')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Further Analysis","metadata":{}},{"cell_type":"code","source":"sale_day_item_level= pd.read_csv(\"../input/memory-optimization-data-manipulation/sale_day_item_level.csv\")\nsale_day_store_level= pd.read_csv(\"../input/memory-optimization-data-manipulation/sale_day_store_level.csv\")\nsale_store_item_level= pd.read_csv(\"../input/memory-optimization-data-manipulation/sale_store_item_level.csv\")","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:49:21.162222Z","iopub.status.idle":"2025-01-30T02:49:21.162470Z","shell.execute_reply.started":"2025-01-30T02:49:21.162340Z","shell.execute_reply":"2025-01-30T02:49:21.162353Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nsale_store_level=sale_day_store_level.groupby(['store_nbr'],as_index=False)['store_sales','item_variety'].agg(['sum'])\n\n\nsale_store_level.columns = sale_store_level.columns.droplevel(1)\nsale_store_level=sale_store_level.reset_index()\nsale_store_level.head()","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:49:21.163170Z","iopub.status.idle":"2025-01-30T02:49:21.163417Z","shell.execute_reply.started":"2025-01-30T02:49:21.163290Z","shell.execute_reply":"2025-01-30T02:49:21.163303Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nsale_item_level=sale_day_item_level.groupby(['item_nbr'],as_index=False)['item_sales'].agg(['sum'])\n\nsale_item_level=sale_item_level.reset_index()\nsale_item_level.head()","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:49:21.164746Z","iopub.status.idle":"2025-01-30T02:49:21.165060Z","shell.execute_reply.started":"2025-01-30T02:49:21.164911Z","shell.execute_reply":"2025-01-30T02:49:21.164928Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntemp=sale_store_level.sort_values('store_sales',ascending=False).reset_index(drop=True)\ntemp=temp.set_index('store_nbr').head(10)\n\nplt.figure(figsize=(12,8))\nsns.barplot(temp.index,temp.store_sales, alpha=0.6, color='blue')\nplt.ylabel('Overall Sales', fontsize=12)\nplt.xlabel('Store Number', fontsize=12)\nplt.title('Top Stores by Overall sale', fontsize=15)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:49:21.165991Z","iopub.status.idle":"2025-01-30T02:49:21.166276Z","shell.execute_reply.started":"2025-01-30T02:49:21.166129Z","shell.execute_reply":"2025-01-30T02:49:21.166145Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntemp1=sale_item_level.sort_values('sum',ascending=False).reset_index(drop=True)\ntemp1=temp1.set_index('item_nbr').head(10)\nplt.figure(figsize=(12,8))\nx=temp1.index.values\ny=temp1['sum'].values\nsns.barplot(x,y, alpha=0.6, color='purple')\nplt.ylabel('Overall Sales', fontsize=12)\nplt.xlabel('Store Number', fontsize=12)\nplt.title('Top Items by Overall sale', fontsize=15)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:49:21.167079Z","iopub.status.idle":"2025-01-30T02:49:21.167328Z","shell.execute_reply.started":"2025-01-30T02:49:21.167199Z","shell.execute_reply":"2025-01-30T02:49:21.167213Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntemp=sale_day_store_level.groupby('Year')['store_sales'].sum()\nplt.figure(figsize=(13,4))\nsns.pointplot(temp.index,temp.values, alpha=0.8)\nplt.ylabel('Overall Sales', fontsize=12)\nplt.xlabel('Year', fontsize=12)\nplt.title('Sale Year Over Year', fontsize=15)\nplt.xticks(rotation='vertical')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:49:21.167945Z","iopub.status.idle":"2025-01-30T02:49:21.168216Z","shell.execute_reply.started":"2025-01-30T02:49:21.168061Z","shell.execute_reply":"2025-01-30T02:49:21.168081Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntemp=sale_day_store_level.groupby(['Year','Month']).aggregate({'store_sales':np.sum,'Year':np.min,'Month':np.min})\ntemp=temp.reset_index(drop=True)\nsns.set(style=\"whitegrid\", color_codes=True)\n\nplt.figure(figsize=(15,8))\nplt.plot(range(1,13),temp.iloc[0:12,0],label=\"2013\")\nplt.plot(range(1,13),temp.iloc[12:24,0],label=\"2014\")\nplt.plot(range(1,13),temp.iloc[24:36,0],label=\"2015\")\nplt.plot(range(1,13),temp.iloc[36:48,0],label=\"2015\")\nplt.ylabel('Overall Sales', fontsize=12)\nplt.xlabel('Month', fontsize=12)\nplt.title('Monthly sales variation', fontsize=15)\nplt.xticks(rotation='vertical')\nplt.legend(['2013', '2014', '2015', '2016'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:49:21.169041Z","iopub.status.idle":"2025-01-30T02:49:21.169317Z","shell.execute_reply.started":"2025-01-30T02:49:21.169168Z","shell.execute_reply":"2025-01-30T02:49:21.169181Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Store Distrubution","metadata":{}},{"cell_type":"code","source":"\nplt.figure(figsize=(15,12))\n\nplt.subplot(221)\n\ntemp = stores['cluster'].value_counts()\n\nsns.barplot(temp.index,temp.values,color=color[5])\nplt.ylabel('Count of stores', fontsize=12)\nplt.xlabel('Cluster', fontsize=12)\nplt.title('Store distribution across cluster', fontsize=15)\n\nplt.subplot(222)\n\ntemp = stores['type'].value_counts()\n\nsns.barplot(temp.index,temp.values,color=color[7])\nplt.ylabel('Count of stores', fontsize=12)\nplt.xlabel('Type of store', fontsize=12)\nplt.title('Store distribution across store types', fontsize=15)\n\nplt.subplot(223)\n\ntemp = stores['state'].value_counts()\n\nsns.barplot(temp.index,temp.values,color=color[8])\nplt.ylabel('Count of stores', fontsize=12)\nplt.xlabel('state', fontsize=12)\nplt.title('Store distribution across states', fontsize=15)\nplt.xticks(rotation='vertical')\n\nplt.subplot(224)\n\ntemp = stores['city'].value_counts()\n\nsns.barplot(temp.index,temp.values,color=color[9])\nplt.ylabel('Count of stores', fontsize=12)\nplt.xlabel('City', fontsize=12)\nplt.title('Store distribution across cities', fontsize=15)\nplt.xticks(rotation='vertical')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:49:21.170248Z","iopub.status.idle":"2025-01-30T02:49:21.170516Z","shell.execute_reply.started":"2025-01-30T02:49:21.170378Z","shell.execute_reply":"2025-01-30T02:49:21.170392Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Sale distribution","metadata":{}},{"cell_type":"code","source":"sale_store_level=sale_store_level.iloc[:,0:2]\n\nmerge=pd.merge(sale_store_level,stores,how='left',on='store_nbr')\n\n\n\nplt.figure(figsize=(15,12))\n\nplt.subplot(221)\n\ntemp = merge.groupby(['cluster'])['store_sales'].sum()\n#plot\nsns.barplot(temp.index,temp.values,color=color[5])\nplt.ylabel('Sales', fontsize=12)\nplt.xlabel('Cluster', fontsize=12)\nplt.title('Cumulative sales across store clusters', fontsize=15)\n\nplt.subplot(222)\n\ntemp = merge.groupby(['type'])['store_sales'].sum()\n#plot\nsns.barplot(temp.index,temp.values,color=color[7])\nplt.ylabel('sales', fontsize=12)\nplt.xlabel('Type of store', fontsize=12)\nplt.title('Cumulative sales across store types', fontsize=15)\n\nplt.subplot(223)\n\ntemp = merge.groupby(['state'])['store_sales'].sum()\n#plot\nsns.barplot(temp.index,temp.values,color=color[8])\nplt.ylabel('sales', fontsize=12)\nplt.xlabel('state', fontsize=12)\nplt.title('Cumulative sales across states', fontsize=15)\nplt.xticks(rotation='vertical')\n\nplt.subplot(224)\n\ntemp = merge.groupby(['city'])['store_sales'].sum()\n#plot\nsns.barplot(temp.index,temp.values,color=color[9])\nplt.ylabel('sales', fontsize=12)\nplt.xlabel('City', fontsize=12)\nplt.title('Cumulative sales across cities', fontsize=15)\nplt.xticks(rotation='vertical')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:49:21.171469Z","iopub.status.idle":"2025-01-30T02:49:21.171730Z","shell.execute_reply.started":"2025-01-30T02:49:21.171594Z","shell.execute_reply":"2025-01-30T02:49:21.171608Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"store_items=pd.merge(sale_store_item_level,items,on='item_nbr')\nstore_items=pd.merge(store_items,stores,on='store_nbr')\nstore_items['item_sales']=store_items['item_sales']\n\n#item\n\ntop_items_by_type=store_items.groupby(['type','item_nbr'])['item_sales'].sum()\ntop_items_by_type=top_items_by_type.reset_index().sort_values(['type','item_sales'],ascending=[True,False])\n\n\ntop_items_by_type=top_items_by_type.groupby(['type']).head(5)\n\n\n\ntop_class_by_type=store_items.groupby(['type','class'])['item_sales'].sum()\ntop_class_by_type=top_class_by_type.reset_index().sort_values(['type','item_sales'],ascending=[True,False])\n\n\ntop_class_by_type=top_class_by_type.groupby(['type']).head(5)\n\n\ntop_family_by_type=store_items.groupby(['type','family'])['item_sales'].sum()\ntop_family_by_type=top_family_by_type.reset_index().sort_values(['type','item_sales'],ascending=[True,False])\n\n\ntop_family_by_type=top_family_by_type.groupby(['type']).head(5)","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:49:21.173854Z","iopub.status.idle":"2025-01-30T02:49:21.174157Z","shell.execute_reply.started":"2025-01-30T02:49:21.174004Z","shell.execute_reply":"2025-01-30T02:49:21.174019Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"top_family_by_type=store_items.groupby(['type','family'])['item_sales'].sum()\ntop_family_by_type=top_family_by_type.reset_index().sort_values(['type','item_sales'],ascending=[True,False])\nx=top_family_by_type.pivot(index='family',columns='type')\ncm = sns.light_palette(\"orange\", as_cmap=True)\nx = x.style.background_gradient(cmap=cm)\nx","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:49:21.175008Z","iopub.status.idle":"2025-01-30T02:49:21.175290Z","shell.execute_reply.started":"2025-01-30T02:49:21.175145Z","shell.execute_reply":"2025-01-30T02:49:21.175160Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"top_items_by_type=store_items.groupby(['type','item_nbr'])['item_sales'].sum()\ntop_items_by_type=top_items_by_type.reset_index().sort_values(['type','item_sales'],ascending=[True,False])\ntop_items_by_type=top_items_by_type.groupby(['item_nbr']).head(20)\n#print(top_items_by_type)\nx=top_items_by_type.pivot(index='item_nbr',columns='type')\nx['total']=x.sum(axis=1)\nx=x.sort_values('total',ascending=False)\ndel(x['total'])\nx=x.head(30)\ncm = sns.light_palette(\"green\", as_cmap=True)\nx = x.style.background_gradient(cmap=cm,axis=1)\nx","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:49:21.176107Z","iopub.status.idle":"2025-01-30T02:49:21.176379Z","shell.execute_reply.started":"2025-01-30T02:49:21.176237Z","shell.execute_reply":"2025-01-30T02:49:21.176252Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler\n\npipe_processing = Pipeline([\n        ('prepare_data', prepare_data()),\n        ('split_data', split_data()),\n        ('process_data', process_data()),\n        ('join_data', join_df())\n    ])\n\n\ndata_df = pipe_processing.fit_transform([train_large, stores, oil, items, transactions, holiday_events])\n\n\nX = data_df.drop(['unit_sales', 'transactions'], axis=1)\nY = data_df[['unit_sales', 'transactions']]","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:49:21.184322Z","iopub.status.idle":"2025-01-30T02:49:21.184624Z","shell.execute_reply.started":"2025-01-30T02:49:21.184471Z","shell.execute_reply":"2025-01-30T02:49:21.184486Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Modelling and testing","metadata":{}},{"cell_type":"markdown","source":"### Generic function for modelling and testing","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression,SGDRegressor,ElasticNet,Ridge\nfrom sklearn.svm import SVC\nfrom sklearn import linear_model\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import mean_squared_error,mean_absolute_error\n\n\ndef checkModelPerformane(model):\n    model.fit(x_train.values, y_train.values)\n    \n    pred = model.predict(x_test.values)\n    \n    print(\"mean_squared_error: \",np.sqrt(mean_squared_error(y_test.values, pred))) \n    print(\"mean_absolute_error: \", np.sqrt(mean_absolute_error(y_test.values, pred)))","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:49:21.186867Z","iopub.status.idle":"2025-01-30T02:49:21.187136Z","shell.execute_reply.started":"2025-01-30T02:49:21.186988Z","shell.execute_reply":"2025-01-30T02:49:21.187012Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Linear regression","metadata":{}},{"cell_type":"code","source":"print(\"LinearRegression\")\ncheckModelPerformane(LinearRegression())","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:49:21.188051Z","iopub.status.idle":"2025-01-30T02:49:21.188317Z","shell.execute_reply.started":"2025-01-30T02:49:21.188176Z","shell.execute_reply":"2025-01-30T02:49:21.188191Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Random forests","metadata":{}},{"cell_type":"code","source":"print(\"Random Forest\")\ncheckModelPerformane(RandomForestRegressor(random_state=42)) ","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:49:21.192848Z","iopub.status.idle":"2025-01-30T02:49:21.193118Z","shell.execute_reply.started":"2025-01-30T02:49:21.192975Z","shell.execute_reply":"2025-01-30T02:49:21.192990Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Random forest model has the lowest error, thus we are going to use it and fine tune it.","metadata":{}},{"cell_type":"markdown","source":"# Model fine tuning","metadata":{}},{"cell_type":"markdown","source":"### Grid search","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\n\nparam_grid = [\n    {'n_estimators': [3, 10, 30], 'max_features': [2, 4, 6, 8]},\n    {'bootstrap': [False], 'n_estimators': [3, 10], 'max_features': [2, 3, 4]},]\n\nforest_reg = RandomForestRegressor(random_state=42)\n \ngrid_search = GridSearchCV(forest_reg, param_grid, cv=5, scoring='neg_mean_squared_error', return_train_score=True)\ngrid_search.fit(x_train.values, y_train.values)","metadata":{"execution":{"iopub.status.busy":"2025-01-30T02:49:21.194184Z","iopub.status.idle":"2025-01-30T02:49:21.194487Z","shell.execute_reply.started":"2025-01-30T02:49:21.194310Z","shell.execute_reply":"2025-01-30T02:49:21.194324Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### find out the best parameters for our model","metadata":{}},{"cell_type":"code","source":"grid_search.best_params_","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:49:21.195327Z","iopub.status.idle":"2025-01-30T02:49:21.195603Z","shell.execute_reply.started":"2025-01-30T02:49:21.195461Z","shell.execute_reply":"2025-01-30T02:49:21.195475Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"grid_search.best_estimator_","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:49:21.197140Z","iopub.status.idle":"2025-01-30T02:49:21.197416Z","shell.execute_reply.started":"2025-01-30T02:49:21.197271Z","shell.execute_reply":"2025-01-30T02:49:21.197285Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Test model on test set","metadata":{}},{"cell_type":"code","source":"!head test.csv","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:49:21.198249Z","iopub.status.idle":"2025-01-30T02:49:21.198522Z","shell.execute_reply.started":"2025-01-30T02:49:21.198376Z","shell.execute_reply":"2025-01-30T02:49:21.198391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_model = grid_search.best_estimator_\n\n\ntest = pd.read_csv(\"../working/test.csv\", parse_dates=['date'])\n\npipe_processing2 = Pipeline([\n        ('split_data', split_data()),\n        ('process_data', process_data()),\n        ('join_data', join_df())\n    ])\n\ntest_df = pipe_processing2.fit_transform(test)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:49:21.199409Z","iopub.status.idle":"2025-01-30T02:49:21.199678Z","shell.execute_reply.started":"2025-01-30T02:49:21.199537Z","shell.execute_reply":"2025-01-30T02:49:21.199551Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:49:21.200898Z","iopub.status.idle":"2025-01-30T02:49:21.201177Z","shell.execute_reply.started":"2025-01-30T02:49:21.201031Z","shell.execute_reply":"2025-01-30T02:49:21.201046Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# final_predictions = final_model.predict(test_x)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-30T02:49:21.201789Z","iopub.status.idle":"2025-01-30T02:49:21.202053Z","shell.execute_reply.started":"2025-01-30T02:49:21.201910Z","shell.execute_reply":"2025-01-30T02:49:21.201924Z"}},"outputs":[],"execution_count":null}]}