{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":7391,"databundleVersionId":44328,"sourceType":"competition"},{"sourceId":84631216,"sourceType":"kernelVersion"}],"dockerImageVersionId":30152,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input/favorita-grocery-sales-forecasting'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-14T14:51:53.162036Z","iopub.execute_input":"2022-01-14T14:51:53.162625Z","iopub.status.idle":"2022-01-14T14:51:53.197798Z","shell.execute_reply.started":"2022-01-14T14:51:53.162519Z","shell.execute_reply":"2022-01-14T14:51:53.196927Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install py7zr","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-01-14T14:51:56.664643Z","iopub.execute_input":"2022-01-14T14:51:56.665418Z","iopub.status.idle":"2022-01-14T14:52:11.732799Z","shell.execute_reply.started":"2022-01-14T14:51:56.665379Z","shell.execute_reply":"2022-01-14T14:52:11.731986Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import py7zr\nfrom subprocess import check_output\n\nfor dirname, _, filenames in os.walk('/kaggle/input/favorita-grocery-sales-forecasting'):\n    for filename in filenames:\n        archive = py7zr.SevenZipFile(os.path.join(dirname, filename), mode='r')\n        archive.extractall(path=\"/kaggle/working\")\n        archive.close()\n\nprint(check_output([\"ls\", \"../working\"]).decode(\"utf8\"))","metadata":{"execution":{"iopub.status.busy":"2022-01-14T14:52:21.924826Z","iopub.execute_input":"2022-01-14T14:52:21.9251Z","iopub.status.idle":"2022-01-14T14:53:56.530727Z","shell.execute_reply.started":"2022-01-14T14:52:21.925072Z","shell.execute_reply":"2022-01-14T14:53:56.528308Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"# Importing the relevant libraries\nimport IPython.display\nimport json\nimport pandas as pd\nimport seaborn as sns\nimport squarify\n%matplotlib inline\nimport missingno as msno\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.tools as tls\nimport numpy as np\nfrom matplotlib import pyplot as plt\ncolor = sns.color_palette()\n\n# D3 modules\nfrom IPython.core.display import display, HTML, Javascript\nfrom string import Template","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.097862Z","iopub.status.idle":"2022-01-09T21:46:19.098209Z","shell.execute_reply.started":"2022-01-09T21:46:19.098027Z","shell.execute_reply":"2022-01-09T21:46:19.098049Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Explore and prepare the data","metadata":{}},{"cell_type":"code","source":"items = pd.read_csv(\"../working/items.csv\")\nholiday_events = pd.read_csv(\"../working/holidays_events.csv\", parse_dates=['date'])\nstores = pd.read_csv(\"../working/stores.csv\")\noil = pd.read_csv(\"../working/oil.csv\", parse_dates=['date'])\ntransactions = pd.read_csv(\"../working/transactions.csv\", parse_dates=['date'])\n# the full training data's output: \"125,497,040 rows | 6 columns\"\n#Therefore I will only load approx 5% of the data just to get a rough idea of what is in store for us.\ntrain = pd.read_csv(\"../working/train.csv\", nrows=6000000  , parse_dates=['date'])\ntrain_large = pd.read_csv('../working/train.csv', skiprows = 115000000, names = train.columns, parse_dates = ['date'])\n","metadata":{"execution":{"iopub.status.busy":"2022-01-14T14:54:03.349183Z","iopub.execute_input":"2022-01-14T14:54:03.349882Z","iopub.status.idle":"2022-01-14T14:54:49.499635Z","shell.execute_reply.started":"2022-01-14T14:54:03.349835Z","shell.execute_reply":"2022-01-14T14:54:49.498682Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-10T17:10:46.726813Z","iopub.execute_input":"2022-01-10T17:10:46.72762Z","iopub.status.idle":"2022-01-10T17:10:46.759922Z","shell.execute_reply.started":"2022-01-10T17:10:46.727575Z","shell.execute_reply":"2022-01-10T17:10:46.758986Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Nulls in Oil columns: {0} => {1}\".format(oil.columns.values,oil.isnull().any().values))\nprint(\"=\"*70)\nprint(\"Nulls in holiday_events columns: {0} => {1}\".format(holiday_events.columns.values,holiday_events.isnull().any().values))\nprint(\"=\"*70)\nprint(\"Nulls in stores columns: {0} => {1}\".format(stores.columns.values,stores.isnull().any().values))\nprint(\"=\"*70)\nprint(\"Nulls in transactions columns: {0} => {1}\".format(transactions.columns.values,transactions.isnull().any().values))","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.10401Z","iopub.status.idle":"2022-01-09T21:46:19.104927Z","shell.execute_reply.started":"2022-01-09T21:46:19.104649Z","shell.execute_reply":"2022-01-09T21:46:19.104679Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":" The only missing data occurs in the oil data file, which provides the historical daily price for oil.","metadata":{}},{"cell_type":"markdown","source":"## Oil Data","metadata":{}},{"cell_type":"code","source":"oil.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.106361Z","iopub.status.idle":"2022-01-09T21:46:19.106858Z","shell.execute_reply.started":"2022-01-09T21:46:19.106651Z","shell.execute_reply":"2022-01-09T21:46:19.106677Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":" trace = go.Scatter(\n     name='Oil prices',\n     x=oil['date'],\n     y=oil['dcoilwtico'].dropna(),\n     mode='lines',\n     line=dict(color='rgb(20, 15, 200, 0.8)'),\n     #fillcolor='rgba(68, 68, 68, 0.3)',\n     fillcolor='rgba(0, 0, 216, 0.3)',\n     fill='tonexty' )\n\n data = [trace]\n\n layout = go.Layout(\n     yaxis=dict(title='Daily Oil price'),\n     title='Daily oil prices from Jan 2013 till July 2017',\n     showlegend = False)\n fig = go.Figure(data=data, layout=layout)\n py.iplot(fig, filename='pandas-time-series-error-bars')","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.108287Z","iopub.status.idle":"2022-01-09T21:46:19.108652Z","shell.execute_reply.started":"2022-01-09T21:46:19.108443Z","shell.execute_reply":"2022-01-09T21:46:19.108467Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"From January 2013 to July 2017, this graph demonstrates that the daily oil price has been on a declining trend. Whereas the price of oil began 2013 by rising and even breaking the $100 barrier for a few months in 2013, the price of oil began to plummet in the middle of 2014, resulting in a significant decline in the price of oil. This trend appears to be true based on some quick open-source research (i.e. Googling), as oil prices were relatively stable from 2010 to mid-2014, after which they drastically fell (due to a confluence of factors including weak demand due to poor economic growth and surging alternative crude oil sources such as shale/tar sands).","metadata":{}},{"cell_type":"markdown","source":"## Stores Data","metadata":{}},{"cell_type":"code","source":"stores.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.110158Z","iopub.status.idle":"2022-01-09T21:46:19.11077Z","shell.execute_reply.started":"2022-01-09T21:46:19.110572Z","shell.execute_reply":"2022-01-09T21:46:19.110595Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"temp=transactions.groupby(['store_nbr']).agg({'date':[np.min,np.max]}).reset_index()\ntemp['store_age']=temp['date']['amax']-temp['date']['amin']\ntemp['open_year']=temp['date']['amin'].dt.year\ndata=temp['open_year'].value_counts()\nplt.figure(figsize=(12,4))\nsns.barplot(data.index,data.values, alpha=0.8, color=color[0])\nplt.ylabel('Stores', fontsize=12)\nplt.xlabel('Store opening Year', fontsize=12)\nplt.title('When were the stores started?', fontsize=15)\nplt.xticks(rotation='vertical')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.111853Z","iopub.status.idle":"2022-01-09T21:46:19.112344Z","shell.execute_reply.started":"2022-01-09T21:46:19.112165Z","shell.execute_reply":"2022-01-09T21:46:19.112186Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"5 Stores were opened in 2015 and 1 each in 2014 and 2017","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(25, 21))\nmarrimeko=stores.city.value_counts().to_frame()\nax = fig.add_subplot(111, aspect=\"equal\")\nax = squarify.plot(sizes=marrimeko['city'].values,label=marrimeko.index,\n              color=sns.color_palette('cubehelix_r', 28), alpha=1)\nax.set_xticks([])\nax.set_yticks([])\nfig=plt.gcf()\nfig.set_size_inches(40,25)\nplt.title(\"Treemap of store counts across different cities\", fontsize=18)\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.113406Z","iopub.status.idle":"2022-01-09T21:46:19.113994Z","shell.execute_reply.started":"2022-01-09T21:46:19.113787Z","shell.execute_reply":"2022-01-09T21:46:19.113811Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig = plt.figure(figsize=(25, 21))\nmarrimeko=stores.state.value_counts().to_frame()\nax = fig.add_subplot(111, aspect=\"equal\")\nax = squarify.plot(sizes=marrimeko['state'].values,label=marrimeko.index,\n              color=sns.color_palette('viridis_r', 28), alpha=1)\nax.set_xticks([])\nax.set_yticks([])\nfig=plt.gcf()\nfig.set_size_inches(40,25)\nplt.title(\"Treemap of store counts across different States\", fontsize=18)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.115447Z","iopub.status.idle":"2022-01-09T21:46:19.116199Z","shell.execute_reply.started":"2022-01-09T21:46:19.115901Z","shell.execute_reply":"2022-01-09T21:46:19.115934Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stores.state.unique()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.117383Z","iopub.status.idle":"2022-01-09T21:46:19.117881Z","shell.execute_reply.started":"2022-01-09T21:46:19.117622Z","shell.execute_reply":"2022-01-09T21:46:19.117658Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Our store numbers will now be arranged against their corresponding shop clusters, allowing us to see whether there are any obvious trends or links in the data. To do so, I'll use the groupby and pivot statements to group our stores Python dataframe based on the values \"store nbr\" and \"cluster.\" Then I'll unstack the grouping by pivoting on the level of store nbr index labels, yielding a DataFrame with a new level of columns that are the store clusters whose inner-most level pertain to the pivoted store nbr index labels. This method is widely used in Python to create stacked barplots, but because we just have unique store nbr numbers, we'll just receive barplots of store numbers arranged by their relevant clusters.","metadata":{}},{"cell_type":"code","source":"neworder = [23, 24, 26, 36, 41, 15, 29, 31, 32, 34, 39, \n            53, 4, 37, 40, 43, 8, 10, 19, 20, 33, 38, 13, \n            21, 2, 6, 7, 3, 22, 25, 27, 28, 30, 35, 42, 44, \n            48, 51, 16, 0, 1, 5, 52, 45, 46, 47, 49, 9, 11, 12, 14, 18, 17, 50]","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.11924Z","iopub.status.idle":"2022-01-09T21:46:19.119736Z","shell.execute_reply.started":"2022-01-09T21:46:19.119449Z","shell.execute_reply":"2022-01-09T21:46:19.119474Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Finally plot the seaborn heatmap\nplt.style.use('dark_background')\nplt.figure(figsize=(15,12))\nstore_pivot = stores.dropna().pivot(\"store_nbr\",\"cluster\", \"store_nbr\")\nax = sns.heatmap(store_pivot, cmap='jet', annot=True, linewidths=0, linecolor='white')\nplt.title('Store numbers and the clusters they are assigned to')","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.121037Z","iopub.status.idle":"2022-01-09T21:46:19.121491Z","shell.execute_reply.started":"2022-01-09T21:46:19.121244Z","shell.execute_reply":"2022-01-09T21:46:19.121271Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":" plt.style.use('seaborn-white')\n nbr_cluster = stores.groupby(['store_nbr','cluster']).size()\n nbr_cluster.unstack().iloc[neworder].plot(kind='bar',stacked=True, colormap= 'tab20', figsize=(13,11),  grid=False)\n plt.title('Store numbers and the clusters they are assigned to', fontsize=14)\n plt.ylabel('')\n plt.xlabel('Store number')\n plt.show()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-01-09T21:46:19.123463Z","iopub.status.idle":"2022-01-09T21:46:19.123961Z","shell.execute_reply.started":"2022-01-09T21:46:19.12371Z","shell.execute_reply":"2022-01-09T21:46:19.123737Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"From visualising the store numbers side-by-side based on the clustering, we can identify certain patterns. \nFor example clusters 3, 6, 10 and 15 are the most common store clusters based off the fact that there are more store_nbrs attributed to them then the others \nwhile on the other end of the spectrum, we have clusters 5 and 17 which are only related to the stores 44 and 51 respectively.","metadata":{}},{"cell_type":"markdown","source":"we can look at the distribution of clusters based on the store type to see if we can identify any apparent relationship between types and the way the company has decided to cluster the particular store.","metadata":{}},{"cell_type":"code","source":"plt.style.use('seaborn-white')\ntype_cluster = stores.groupby(['type','cluster']).size()\ntype_cluster.unstack().plot(kind='bar',stacked=True, colormap= 'PuBu', figsize=(13,11),  grid=False)\nplt.title('Stacked Barplot of Store types and their cluster distribution', fontsize=18)\nplt.ylabel('Count of clusters in a particular store type', fontsize=16)\nplt.xlabel('Store type', fontsize=16)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.125384Z","iopub.status.idle":"2022-01-09T21:46:19.125879Z","shell.execute_reply.started":"2022-01-09T21:46:19.12567Z","shell.execute_reply":"2022-01-09T21:46:19.125701Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.style.use('seaborn-white')\ncity_cluster = stores.groupby(['city','type']).store_nbr.size()\ncity_cluster.unstack().plot(kind='bar',stacked=True, colormap= 'viridis', figsize=(13,11),  grid=False)\nplt.title('Stacked Barplot of Store types opened for each city')\nplt.ylabel('Count of stores for a particular city')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.126713Z","iopub.status.idle":"2022-01-09T21:46:19.127031Z","shell.execute_reply.started":"2022-01-09T21:46:19.126855Z","shell.execute_reply":"2022-01-09T21:46:19.126878Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Guayaquil and Quito are two cities that stand out in terms of the range of retail kinds available. These are unsurprising given that Quito is Ecuador's capital and Guayaquil is the country's largest and most populated metropolis. As a result, one might expect Corporacion Favorita to target these major cities with the most diverse store types, as evidenced by the highest counts of store nbrs attributed to those two cities.","metadata":{}},{"cell_type":"markdown","source":"## Holiday Events Data","metadata":{}},{"cell_type":"code","source":"holiday_events.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.128168Z","iopub.status.idle":"2022-01-09T21:46:19.128504Z","shell.execute_reply.started":"2022-01-09T21:46:19.128326Z","shell.execute_reply":"2022-01-09T21:46:19.128351Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#does the transactions peak at holiday events?\nplt.figure(figsize=(12,12))\nplt.plot(transactions.rolling(window=30,center=False).mean(),label='Rolling Mean');\nplt.plot(transactions.rolling(window=30,center=False).std(),label='Rolling sd');\nplt.legend();","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.130232Z","iopub.status.idle":"2022-01-09T21:46:19.13082Z","shell.execute_reply.started":"2022-01-09T21:46:19.130622Z","shell.execute_reply":"2022-01-09T21:46:19.130646Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.style.use('seaborn-white')\nholiday_local_type = holiday_events.groupby(['locale_name', 'type']).size()\nholiday_local_type.unstack().plot(kind='bar',stacked=True, colormap= 'magma_r', figsize=(12,10),  grid=False)\nplt.title('Stacked Barplot of locale name against event type')\nplt.ylabel('Count of entries')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.131987Z","iopub.status.idle":"2022-01-09T21:46:19.132307Z","shell.execute_reply.started":"2022-01-09T21:46:19.132134Z","shell.execute_reply":"2022-01-09T21:46:19.132156Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### D3.js visualization library","metadata":{}},{"cell_type":"code","source":" with pd.option_context('display.max_rows', None, 'display.max_columns', 3):\n     print(holiday_events[['type','description']].apply(pd.Series.value_counts))","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-01-09T21:46:19.133862Z","iopub.status.idle":"2022-01-09T21:46:19.13462Z","shell.execute_reply.started":"2022-01-09T21:46:19.134391Z","shell.execute_reply":"2022-01-09T21:46:19.134418Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prepping the json file\nholiday_json = {\n\"name\": \"flare\",\n\"children\": [\n{\n\"name\": \"Additional\",\n\"children\":[\n{\"name\": \"Batalla de Pichincha\",       \"size\": 5.0},\n{\"name\": \"Cantonizacion de Cayambe\",   \"size\": 6.0},\n{\"name\": \"Cantonizacion de El Carmen\", \"size\": 6.0},\n{\"name\": \"Cantonizacion de Guaranda\",  \"size\": 6.0},\n{\"name\": \"Cantonizacion de Latacunga\", \"size\": 6.0},\n{\"name\": \"Cantonizacion de Libertad\",  \"size\": 6.0},\n{\"name\": \"Cantonizacion de Quevedo\",   \"size\": 6.0},\n{\"name\": \"Cantonizacion de Riobamba\",  \"size\": 6.0},\n{\"name\": \"Cantonizacion de Salinas\",   \"size\": 6.0},\n{\"name\": \"Cantonizacion del Puyo\",     \"size\": 6.0},\n{\"name\": \"Carnaval\",                   \"size\": 0.0},\n{\"name\": \"Dia de Difuntos\",            \"size\": 6.0},\n{\"name\": \"Dia de la Madre\",            \"size\": 5.0},\n{\"name\": \"Dia de la Madre-1\",          \"size\": 5.0},\n{\"name\": \"Dia del Trabajo\",             \"size\": 5.0},\n{\"name\": \"Fundacion de Guayaquil\",    \"size\": 5.0},\n{\"name\": \"Fundacion de Guayaquil-1\",  \"size\": 5.0},\n{\"name\": \"Fundacion de Quito\",        \"size\": 6.0},\n{\"name\": \"Fundacion de Quito-1\",      \"size\": 6.0},\n{\"name\": \"Navidad+1                                      \", \"size\": 6.0},\n{\"name\": \"Navidad-1                                      \", \"size\": 6.0},\n{\"name\": \"Navidad-2                                      \", \"size\": 6.0},\n{\"name\": \"Navidad-3                                      \", \"size\": 6.0},\n{\"name\": \"Navidad-4                                      \", \"size\": 6.0},\n]\n},\n{\n\"name\":  \"Holiday\",\n\"children\":[\n{\"name\": \"Fundacion de Ambato\",       \"size\": 6.0},\n{\"name\": \"Fundacion de Cuenca\",       \"size\": 7.0},\n{\"name\": \"Fundacion de Esmeraldas\",   \"size\": 6.0},\n{\"name\": \"Fundacion de Ibarra\",       \"size\": 7.0},\n{\"name\": \"Fundacion de Loja\",         \"size\": 6.0},\n{\"name\": \"Fundacion de Machala\",      \"size\": 6.0},\n{\"name\": \"Fundacion de Manta\",        \"size\": 6.0},\n{\"name\": \"Fundacion de Riobamba\",     \"size\": 6.0},\n{\"name\": \"Fundacion de Santo Domingo\", \"size\": 6.0}\n]\n},\n{\n\"name\": \"Event\",\n\"children\": [\n{\"name\": \"Inauguracion Mundial de futbol Brasil          \", \"size\": 1.0},\n{\"name\": \"Independencia de Ambato                        \", \"size\": 6.0},\n{\"name\": \"Independencia de Cuenca                        \", \"size\": 6.0},\n{\"name\": \"Independencia de Guaranda                      \", \"size\": 6.0},\n{\"name\": \"Independencia de Guayaquil                     \", \"size\": 6.0},\n{\"name\": \"Independencia de Latacunga                     \", \"size\": 6.0},\n{\"name\": \"Mundial de futbol Brasil: Cuartos de Final     \", \"size\": 2.0},\n{\"name\": \"Mundial de futbol Brasil: Ecuador-Francia      \", \"size\": 1.0},\n{\"name\": \"Mundial de futbol Brasil: Ecuador-Honduras     \", \"size\": 1.0},\n{\"name\": \"Mundial de futbol Brasil: Ecuador-Suiza        \", \"size\": 1.0},\n{\"name\": \"Mundial de futbol Brasil: Final                \", \"size\": 1.0},\n{\"name\": \"Mundial de futbol Brasil: Octavos de Final     \", \"size\": 4.0},\n{\"name\": \"Mundial de futbol Brasil: Semifinales          \", \"size\": 2.0},\n{\"name\": \"Mundial de futbol Brasil: Tercer y cuarto lugar\", \"size\": 1.0},\n{\"name\": \"Navidad                                        \", \"size\": 6.0},\n{\"name\": \"Primer Grito de Independencia                  \", \"size\": 6.0},\n{\"name\": \"Primer dia del ano                             \", \"size\": 5.0},\n{\"name\": \"Primer dia del ano-1                           \", \"size\": 5.0},\n{\"name\": \"Black Friday\",               \"size\": 3.0},\n{\"name\": \"Cyber Monday\",               \"size\": 3.0},\n{\"name\": \"Provincializacion Santa Elena                  \", \"size\": 6.0},\n{\"name\": \"Provincializacion de Cotopaxi                  \", \"size\": 6.0},\n{\"name\": \"Provincializacion de Imbabura                  \", \"size\": 6.0},\n{\"name\": \"Provincializacion de Santo Domingo             \", \"size\": 6.0},\n{\"name\": \"Terremoto Manabi                               \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+1                             \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+10                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+11                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+12                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+13                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+14                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+15                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+16                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+17                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+18                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+19                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+2                             \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+20                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+21                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+22                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+23                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+24                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+25                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+26                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+27                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+28                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+29                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+3                             \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+30                            \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+4                             \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+5                             \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+6                             \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+7                             \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+8                             \", \"size\": 1.0},\n{\"name\": \"Terremoto Manabi+9                             \", \"size\": 1.0}\n]\n},\n{\n\"name\": \"Transfer\",\n\"children\":[\n{\"name\": \"Traslado Batalla de Pichincha         \", \"size\": 2.0},\n{\"name\": \"Traslado Fundacion de Guayaquil       \", \"size\": 1.0},\n{\"name\": \"Traslado Fundacion de Quito           \", \"size\": 1.0},\n{\"name\": \"Traslado Independencia de Guayaquil   \", \"size\": 3.0},\n{\"name\": \"Traslado Primer Grito de Independencia\", \"size\": 2.0},\n{\"name\": \"Traslado Primer dia del ano           \", \"size\": 1.0},\n{\"name\": \"Viernes Santo                         \", \"size\": 5.0}\n]\n},\n    {\n\"name\": \"Bridge\",\n\"children\":[\n{\"name\": \"Puente Dia de Difuntos                         \", \"size\": 1.0},\n{\"name\": \"Puente Navidad                                 \", \"size\": 2.0},\n{\"name\": \"Puente Primer dia del ano                      \", \"size\": 2.0},\n]\n},\n{\n\"name\": \"Work Day\",\n\"children\":[\n    {\"name\": \"Recupero puente Navidad\", \"size\": 2.0},\n    {\"name\": \"ecupero puente primer dia del ano\", \"size\": 2.0},\n    {\"name\": \"Recupero Puente Navidad\", \"size\": 2.0},\n    {\"name\": \"Recupero Puente Primer dia del ano\", \"size\": 2.0},\n    {\"name\": \"Recupero Puente Dia de Difuntos\", \"size\": 2.0}\n]\n}\n] \n} ","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-09T21:46:19.136137Z","iopub.status.idle":"2022-01-09T21:46:19.13647Z","shell.execute_reply.started":"2022-01-09T21:46:19.136295Z","shell.execute_reply":"2022-01-09T21:46:19.136319Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# dumping the holiday_events data into a json file\nwith open('output.json', 'w') as outfile:  \n    json.dump(holiday_json, outfile)\npd.read_json('output.json').head()\n\n#Embedding the html string\nhtml_string = \"\"\"\n<!DOCTYPE html>\n<meta charset=\"utf-8\">\n<style>\n\n.node {\n  cursor: pointer;\n}\n\n.node:hover {\n  stroke: #000;\n  stroke-width: 1.5px;\n}\n\n.node--leaf {\n  fill: white;\n}\n\n.label {\n  font: 11px \"Helvetica Neue\", Helvetica, Arial, sans-serif;\n  text-anchor: middle;\n  text-shadow: 0 1px 0 #fff, 1px 0 0 #fff, -1px 0 0 #fff, 0 -1px 0 #fff;\n}\n\n.label,\n.node--root,\n.node--leaf {\n  pointer-events: none;\n}\n\n</style>\n<svg width=\"760\" height=\"760\"></svg>\n\"\"\"","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-09T21:46:19.137394Z","iopub.status.idle":"2022-01-09T21:46:19.137736Z","shell.execute_reply.started":"2022-01-09T21:46:19.137554Z","shell.execute_reply":"2022-01-09T21:46:19.137576Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Finally embed the D3.js to produce the circular treemap\njs_string=\"\"\"\n require.config({\n    paths: {\n        d3: \"https://d3js.org/d3.v4.min\"\n     }\n });\n\n  require([\"d3\"], function(d3) {\n\n   console.log(d3);\n\nvar svg = d3.select(\"svg\"),\n    margin = 20,\n    diameter = +svg.attr(\"width\"),\n    g = svg.append(\"g\").attr(\"transform\", \"translate(\" + diameter / 2 + \",\" + diameter / 2 + \")\");\n\nvar color = d3.scaleSequential(d3.interpolateViridis)\n    .domain([-4, 4]);\n\nvar pack = d3.pack()\n    .size([diameter - margin, diameter - margin])\n    .padding(2);\n\nd3.json(\"output.json\", function(error, root) {\n  if (error) throw error;\n\n  root = d3.hierarchy(root)\n      .sum(function(d) { return d.size; })\n      .sort(function(a, b) { return b.value - a.value; });\n\n  var focus = root,\n      nodes = pack(root).descendants(),\n      view;\n\n  var circle = g.selectAll(\"circle\")\n    .data(nodes)\n    .enter().append(\"circle\")\n      .attr(\"class\", function(d) { return d.parent ? d.children ? \"node\" : \"node node--leaf\" : \"node node--root\"; })\n      .style(\"fill\", function(d) { return d.children ? color(d.depth) : null; })\n      .on(\"click\", function(d) { if (focus !== d) zoom(d), d3.event.stopPropagation(); });\n\n  var text = g.selectAll(\"text\")\n    .data(nodes)\n    .enter().append(\"text\")\n      .attr(\"class\", \"label\")\n      .style(\"fill-opacity\", function(d) { return d.parent === root ? 1 : 0; })\n      .style(\"display\", function(d) { return d.parent === root ? \"inline\" : \"none\"; })\n      .text(function(d) { return d.data.name; });\n\n  var node = g.selectAll(\"circle,text\");\n\n  svg\n      .style(\"background\", color(-1))\n      .on(\"click\", function() { zoom(root); });\n\n  zoomTo([root.x, root.y, root.r * 2 + margin]);\n\n  function zoom(d) {\n    var focus0 = focus; focus = d;\n\n    var transition = d3.transition()\n        .duration(d3.event.altKey ? 7500 : 750)\n        .tween(\"zoom\", function(d) {\n          var i = d3.interpolateZoom(view, [focus.x, focus.y, focus.r * 2 + margin]);\n          return function(t) { zoomTo(i(t)); };\n        });\n\n    transition.selectAll(\"text\")\n      .filter(function(d) { return d.parent === focus || this.style.display === \"inline\"; })\n        .style(\"fill-opacity\", function(d) { return d.parent === focus ? 1 : 0; })\n        .on(\"start\", function(d) { if (d.parent === focus) this.style.display = \"inline\"; })\n        .on(\"end\", function(d) { if (d.parent !== focus) this.style.display = \"none\"; });\n  }\n\n  function zoomTo(v) {\n    var k = diameter / v[2]; view = v;\n    node.attr(\"transform\", function(d) { return \"translate(\" + (d.x - v[0]) * k + \",\" + (d.y - v[1]) * k + \")\"; });\n    circle.attr(\"r\", function(d) { return d.r * k; });\n  }\n});\n  });\n \"\"\"","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-09T21:46:19.139492Z","iopub.status.idle":"2022-01-09T21:46:19.140054Z","shell.execute_reply.started":"2022-01-09T21:46:19.139848Z","shell.execute_reply":"2022-01-09T21:46:19.139873Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"h = display(HTML(html_string))\nj = IPython.display.Javascript(js_string)\nIPython.display.display_javascript(j)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.141299Z","iopub.status.idle":"2022-01-09T21:46:19.141663Z","shell.execute_reply.started":"2022-01-09T21:46:19.141459Z","shell.execute_reply":"2022-01-09T21:46:19.141483Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"holiday_events.type.unique()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.142764Z","iopub.status.idle":"2022-01-09T21:46:19.143082Z","shell.execute_reply.started":"2022-01-09T21:46:19.142912Z","shell.execute_reply":"2022-01-09T21:46:19.142934Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Transactions data","metadata":{}},{"cell_type":"code","source":"transactions.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.144238Z","iopub.status.idle":"2022-01-09T21:46:19.144595Z","shell.execute_reply.started":"2022-01-09T21:46:19.144399Z","shell.execute_reply":"2022-01-09T21:46:19.144421Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.style.use('seaborn-white')\nplt.figure(figsize=(13,11))\nplt.plot(transactions.date.values, transactions.transactions.values, color='darkblue')\nplt.ylim(-50, 10000)\nplt.title(\"Distribution of transactions per day from 2013 till 2017\")\nplt.ylabel('transactions per day', fontsize= 16)\nplt.xlabel('Date', fontsize= 16)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.146334Z","iopub.status.idle":"2022-01-09T21:46:19.147129Z","shell.execute_reply.started":"2022-01-09T21:46:19.146842Z","shell.execute_reply":"2022-01-09T21:46:19.146877Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The bigger yearly periodic spike in transactions seem to occur at the end of the year in December. Perhaps this is due to some sort of Christmas sale/discount that Corporacion Favorita holds every December.","metadata":{}},{"cell_type":"code","source":"#transactions\n# month over month sales\ntransactions['date']=pd.to_datetime(transactions['date'])\ntemp=transactions.groupby(['date']).aggregate({'store_nbr':'count','transactions':np.sum})\ntemp=temp.reset_index()\ntemp_2013=temp[temp['date'].dt.year==2013].reset_index(drop=True)\ntemp_2014=temp[temp['date'].dt.year==2014].reset_index(drop=True)\ntemp_2015=temp[temp['date'].dt.year==2015].reset_index(drop=True)\ntemp_2016=temp[temp['date'].dt.year==2016].reset_index(drop=True)\ntemp_2017=temp[temp['date'].dt.year==2017].reset_index(drop=True)\n\nsns.set(style=\"whitegrid\", color_codes=True)\nplt.figure(figsize=(15,14))\nplt.subplot(211)\nplt.plot(temp_2013['date'],temp_2013.iloc[:,1],label=\"2013\")\nplt.plot(temp_2014['date'],temp_2014.iloc[:,1],label=\"2014\")\nplt.plot(temp_2015['date'],temp_2015.iloc[:,1],label=\"2015\")\nplt.plot(temp_2016['date'],temp_2016.iloc[:,1],label=\"2016\")\nplt.plot(temp_2017['date'],temp_2017.iloc[:,1],label=\"2017\")\nplt.ylabel('Number of stores open', fontsize=12)\nplt.xlabel('Time', fontsize=12)\nplt.title('Number of stores open', fontsize=15)\nplt.xticks(rotation='vertical')\nplt.legend(['2013', '2014', '2015', '2016'], loc='lower right')\n\nplt.subplot(212)\nplt.plot(temp_2013.index,temp_2013.iloc[:,1],label=\"2013\")\nplt.plot(temp_2014.index,temp_2014.iloc[:,1],label=\"2014\")\nplt.plot(temp_2015.index,temp_2015.iloc[:,1],label=\"2015\")\nplt.plot(temp_2016.index,temp_2016.iloc[:,1],label=\"2016\")\nplt.plot(temp_2017.index,temp_2017.iloc[:,1],label=\"2017\")\n\n\nplt.ylabel('Number of stores open', fontsize=12)\nplt.xlabel('Day of year', fontsize=12)\nplt.title('Number of stores open', fontsize=15)\nplt.xticks(rotation='vertical')\nplt.legend(['2013', '2014', '2015', '2016'], loc='lower right')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.148601Z","iopub.status.idle":"2022-01-09T21:46:19.149133Z","shell.execute_reply.started":"2022-01-09T21:46:19.148925Z","shell.execute_reply":"2022-01-09T21:46:19.148952Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"There seems to be certain local holidays where some of the stores are closed. But there is no consistent pattern of holidays where stores are closed","metadata":{}},{"cell_type":"markdown","source":"### Store #47","metadata":{}},{"cell_type":"markdown","source":"The chain established itselft in Quito in 1952 (We knew this from Wikipedia), so let's pick a shop in Quito as a starting point, as the brand is well established there. Let us pick #47 and plot the corresponding transactions time series. With a well established store, we can predict that the time series will be almost stationary. High seasonality is expected too, as people consume more during celebration periods.","metadata":{}},{"cell_type":"code","source":"ts=transactions.loc[transactions['store_nbr']==47,['date','transactions']].set_index('date')\nts=ts.transactions.astype('float')\nplt.figure(figsize=(12,12))\nplt.title('Daily transactions in store #47')\nplt.xlabel('time')\nplt.ylabel('Number of transactions')\nplt.plot(ts);","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.150249Z","iopub.status.idle":"2022-01-09T21:46:19.150597Z","shell.execute_reply.started":"2022-01-09T21:46:19.150396Z","shell.execute_reply":"2022-01-09T21:46:19.150418Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Items data","metadata":{}},{"cell_type":"code","source":"items.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.151689Z","iopub.status.idle":"2022-01-09T21:46:19.152327Z","shell.execute_reply.started":"2022-01-09T21:46:19.152085Z","shell.execute_reply":"2022-01-09T21:46:19.152115Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x, y = (list(x) for x in zip(*sorted(zip(items.family.value_counts().index, \n                                         items.family.value_counts().values), \n                                        reverse = False)))\ntrace2 = go.Bar(\n    y=items.family.value_counts().values,\n    x=items.family.value_counts().index,\n    marker=dict(\n        color=items.family.value_counts().values,\n        colorscale = 'Portland',\n        reversescale = False\n    ),\n    orientation='v',\n)\n\nlayout = dict(\n    title='Counts of items per family category',\n     width = 800, height = 800,\n    yaxis=dict(\n        showgrid=False,\n        showline=False,\n        showticklabels=True,\n    ))\n\nfig1 = go.Figure(data=[trace2])\nfig1['layout'].update(layout)\npy.iplot(fig1, filename='plots')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-01-09T21:46:19.153539Z","iopub.status.idle":"2022-01-09T21:46:19.154233Z","shell.execute_reply.started":"2022-01-09T21:46:19.15397Z","shell.execute_reply":"2022-01-09T21:46:19.154001Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"As we can see from the plot, the top 3 family categories are the GROCERY I, BEVERAGES and CLEANING categories.","metadata":{}},{"cell_type":"code","source":"x, y = (list(x) for x in zip(*sorted(zip(items['class'].value_counts().index, \n                                         items['class'].value_counts().values), \n                                        reverse = False)))\ntrace2 = go.Bar(\n    x=items['class'].value_counts().index,\n    y=items['class'].value_counts().values,\n    marker=dict(\n        color=items['class'].value_counts().values,\n        colorscale = 'Jet',\n        reversescale = True\n    ),\n    orientation='v',\n)\n\nlayout = dict(\n    title='Number of items attributed to a particular item class',\n     width = 800, height = 1400,\n    yaxis=dict(\n        showgrid=False,\n        showline=False,\n        showticklabels=True\n    ))\n\nfig1 = go.Figure(data=[trace2])\nfig1['layout'].update(layout)\npy.iplot(fig1, filename='plots')","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-01-09T21:46:19.155429Z","iopub.status.idle":"2022-01-09T21:46:19.155788Z","shell.execute_reply.started":"2022-01-09T21:46:19.15561Z","shell.execute_reply":"2022-01-09T21:46:19.155634Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.style.use('seaborn-white')\nfam_perishable = items.groupby(['family', 'perishable']).size()\nfam_perishable.unstack().plot(kind='bar',stacked=True, colormap= 'coolwarm', figsize=(12,10),  grid=False)\nplt.title('Stacked Barplot of locale name against event type')\nplt.ylabel('Count of entries')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.157117Z","iopub.status.idle":"2022-01-09T21:46:19.157456Z","shell.execute_reply.started":"2022-01-09T21:46:19.157285Z","shell.execute_reply":"2022-01-09T21:46:19.157307Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training Data","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.158623Z","iopub.status.idle":"2022-01-09T21:46:19.15895Z","shell.execute_reply.started":"2022-01-09T21:46:19.158782Z","shell.execute_reply":"2022-01-09T21:46:19.158803Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.style.use('seaborn-deep')\nplt.figure(figsize=(13,11))\nplt.plot(train.date.values, train.unit_sales)\nplt.ylim(-50, 10000)\nplt.ylabel('transactions per day')\nplt.xlabel('Date')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.160858Z","iopub.status.idle":"2022-01-09T21:46:19.161177Z","shell.execute_reply.started":"2022-01-09T21:46:19.161008Z","shell.execute_reply":"2022-01-09T21:46:19.161032Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Further Analysis","metadata":{}},{"cell_type":"markdown","source":"Further to make EDA easier, we used data which rolled up the sales to different levels\n\n - Day-Store level\n - Day-Item level\n - Store level\n - Item level\n - Day level","metadata":{}},{"cell_type":"code","source":"sale_day_item_level= pd.read_csv(\"../input/memory-optimization-data-manipulation/sale_day_item_level.csv\")\nsale_day_store_level= pd.read_csv(\"../input/memory-optimization-data-manipulation/sale_day_store_level.csv\")\nsale_store_item_level= pd.read_csv(\"../input/memory-optimization-data-manipulation/sale_store_item_level.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.162614Z","iopub.status.idle":"2022-01-09T21:46:19.162946Z","shell.execute_reply.started":"2022-01-09T21:46:19.162773Z","shell.execute_reply":"2022-01-09T21:46:19.162797Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Creating store level metrics\nsale_store_level=sale_day_store_level.groupby(['store_nbr'],as_index=False)['store_sales','item_variety'].agg(['sum'])\n\n# Here the group by gives a multiindex , removing that\nsale_store_level.columns = sale_store_level.columns.droplevel(1)\nsale_store_level=sale_store_level.reset_index()\nsale_store_level.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.164167Z","iopub.status.idle":"2022-01-09T21:46:19.16449Z","shell.execute_reply.started":"2022-01-09T21:46:19.164319Z","shell.execute_reply":"2022-01-09T21:46:19.164342Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Creating item level metrics\nsale_item_level=sale_day_item_level.groupby(['item_nbr'],as_index=False)['item_sales'].agg(['sum'])\n\nsale_item_level=sale_item_level.reset_index()\nsale_item_level.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.165552Z","iopub.status.idle":"2022-01-09T21:46:19.165894Z","shell.execute_reply.started":"2022-01-09T21:46:19.165723Z","shell.execute_reply":"2022-01-09T21:46:19.165742Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Sorting by sales\ntemp=sale_store_level.sort_values('store_sales',ascending=False).reset_index(drop=True)\ntemp=temp.set_index('store_nbr').head(10)\n\nplt.figure(figsize=(12,8))\nsns.barplot(temp.index,temp.store_sales, alpha=0.6, color='blue')\nplt.ylabel('Overall Sales', fontsize=12)\nplt.xlabel('Store Number', fontsize=12)\nplt.title('Top Stores by Overall sale', fontsize=15)\n# plt.xticks(rotation='vertical')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.166956Z","iopub.status.idle":"2022-01-09T21:46:19.167279Z","shell.execute_reply.started":"2022-01-09T21:46:19.167104Z","shell.execute_reply":"2022-01-09T21:46:19.167127Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Sorting by sales\ntemp1=sale_item_level.sort_values('sum',ascending=False).reset_index(drop=True)\ntemp1=temp1.set_index('item_nbr').head(10)\nplt.figure(figsize=(12,8))\nx=temp1.index.values\ny=temp1['sum'].values\nsns.barplot(x,y, alpha=0.6, color='purple')\nplt.ylabel('Overall Sales', fontsize=12)\nplt.xlabel('Store Number', fontsize=12)\nplt.title('Top Items by Overall sale', fontsize=15)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.168744Z","iopub.status.idle":"2022-01-09T21:46:19.169062Z","shell.execute_reply.started":"2022-01-09T21:46:19.168891Z","shell.execute_reply":"2022-01-09T21:46:19.168914Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#YOY sales\ntemp=sale_day_store_level.groupby('Year')['store_sales'].sum()\nplt.figure(figsize=(13,4))\nsns.pointplot(temp.index,temp.values, alpha=0.8)\nplt.ylabel('Overall Sales', fontsize=12)\nplt.xlabel('Year', fontsize=12)\nplt.title('Sale Year Over Year', fontsize=15)\nplt.xticks(rotation='vertical')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.17025Z","iopub.status.idle":"2022-01-09T21:46:19.170597Z","shell.execute_reply.started":"2022-01-09T21:46:19.170397Z","shell.execute_reply":"2022-01-09T21:46:19.170421Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# month over month sales\ntemp=sale_day_store_level.groupby(['Year','Month']).aggregate({'store_sales':np.sum,'Year':np.min,'Month':np.min})\ntemp=temp.reset_index(drop=True)\nsns.set(style=\"whitegrid\", color_codes=True)\n# temp\nplt.figure(figsize=(15,8))\nplt.plot(range(1,13),temp.iloc[0:12,0],label=\"2013\")\nplt.plot(range(1,13),temp.iloc[12:24,0],label=\"2014\")\nplt.plot(range(1,13),temp.iloc[24:36,0],label=\"2015\")\nplt.plot(range(1,13),temp.iloc[36:48,0],label=\"2015\")\nplt.ylabel('Overall Sales', fontsize=12)\nplt.xlabel('Month', fontsize=12)\nplt.title('Monthly sales variation', fontsize=15)\nplt.xticks(rotation='vertical')\nplt.legend(['2013', '2014', '2015', '2016'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.172581Z","iopub.status.idle":"2022-01-09T21:46:19.172915Z","shell.execute_reply.started":"2022-01-09T21:46:19.172738Z","shell.execute_reply":"2022-01-09T21:46:19.172762Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Store Distrubution","metadata":{}},{"cell_type":"code","source":"#Count of stores in different types and clusters\nplt.figure(figsize=(15,12))\n#row col plotnumber - 121\nplt.subplot(221)\n# Count of stores for each type \ntemp = stores['cluster'].value_counts()\n#plot\nsns.barplot(temp.index,temp.values,color=color[5])\nplt.ylabel('Count of stores', fontsize=12)\nplt.xlabel('Cluster', fontsize=12)\nplt.title('Store distribution across cluster', fontsize=15)\n\nplt.subplot(222)\n# Count of stores for each type \ntemp = stores['type'].value_counts()\n#plot\nsns.barplot(temp.index,temp.values,color=color[7])\nplt.ylabel('Count of stores', fontsize=12)\nplt.xlabel('Type of store', fontsize=12)\nplt.title('Store distribution across store types', fontsize=15)\n\nplt.subplot(223)\n# Count of stores for each type \ntemp = stores['state'].value_counts()\n#plot\nsns.barplot(temp.index,temp.values,color=color[8])\nplt.ylabel('Count of stores', fontsize=12)\nplt.xlabel('state', fontsize=12)\nplt.title('Store distribution across states', fontsize=15)\nplt.xticks(rotation='vertical')\n\nplt.subplot(224)\n# Count of stores for each type \ntemp = stores['city'].value_counts()\n#plot\nsns.barplot(temp.index,temp.values,color=color[9])\nplt.ylabel('Count of stores', fontsize=12)\nplt.xlabel('City', fontsize=12)\nplt.title('Store distribution across cities', fontsize=15)\nplt.xticks(rotation='vertical')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.174342Z","iopub.status.idle":"2022-01-09T21:46:19.174701Z","shell.execute_reply.started":"2022-01-09T21:46:19.174495Z","shell.execute_reply":"2022-01-09T21:46:19.17454Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Sale distribution","metadata":{}},{"cell_type":"code","source":"sale_store_level=sale_store_level.iloc[:,0:2]\n#print(sale_store_level)\nmerge=pd.merge(sale_store_level,stores,how='left',on='store_nbr')\n#temp\n\n#Sale of stores in different types and clusters\nplt.figure(figsize=(15,12))\n#row col plotnumber - 121\nplt.subplot(221)\n# Sale of stores for each type \ntemp = merge.groupby(['cluster'])['store_sales'].sum()\n#plot\nsns.barplot(temp.index,temp.values,color=color[5])\nplt.ylabel('Sales', fontsize=12)\nplt.xlabel('Cluster', fontsize=12)\nplt.title('Cumulative sales across store clusters', fontsize=15)\n\nplt.subplot(222)\n# sale of stores for each type \ntemp = merge.groupby(['type'])['store_sales'].sum()\n#plot\nsns.barplot(temp.index,temp.values,color=color[7])\nplt.ylabel('sales', fontsize=12)\nplt.xlabel('Type of store', fontsize=12)\nplt.title('Cumulative sales across store types', fontsize=15)\n\nplt.subplot(223)\n# sale of stores for each type \ntemp = merge.groupby(['state'])['store_sales'].sum()\n#plot\nsns.barplot(temp.index,temp.values,color=color[8])\nplt.ylabel('sales', fontsize=12)\nplt.xlabel('state', fontsize=12)\nplt.title('Cumulative sales across states', fontsize=15)\nplt.xticks(rotation='vertical')\n\nplt.subplot(224)\n# sale of stores for city\ntemp = merge.groupby(['city'])['store_sales'].sum()\n#plot\nsns.barplot(temp.index,temp.values,color=color[9])\nplt.ylabel('sales', fontsize=12)\nplt.xlabel('City', fontsize=12)\nplt.title('Cumulative sales across cities', fontsize=15)\nplt.xticks(rotation='vertical')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.17607Z","iopub.status.idle":"2022-01-09T21:46:19.176396Z","shell.execute_reply.started":"2022-01-09T21:46:19.176217Z","shell.execute_reply":"2022-01-09T21:46:19.176247Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Sale variation","metadata":{}},{"cell_type":"code","source":"sale_store_level=sale_store_level.iloc[:,0:2]\nmerge=pd.merge(sale_store_level,stores,how='left',on='store_nbr')\n\nplt.figure(figsize=(15,12))\n#row col plotnumber - 121\nplt.subplot(221)\n#plot\nsns.boxplot(x='cluster', y=\"store_sales\", data=merge)\nplt.ylabel('Sales', fontsize=12)\nplt.xlabel('Cluster', fontsize=12)\nplt.title('Variation across store clusters', fontsize=15)\n\nplt.subplot(222)\n# sale of stores for each type \nsns.boxplot(x='type', y=\"store_sales\", data=merge)\nplt.ylabel('sales', fontsize=12)\nplt.xlabel('Type of store', fontsize=12)\nplt.title('Variation across store types', fontsize=15)\n\nplt.subplot(223)\n# sale of stores for each type \nsns.boxplot(x='state', y=\"store_sales\", data=merge)\nplt.ylabel('sales', fontsize=12)\nplt.xlabel('state', fontsize=12)\nplt.title('Variation across states', fontsize=15)\nplt.xticks(rotation='vertical')\n\nplt.subplot(224)\n# sale of stores for city\nsns.boxplot(x='city', y=\"store_sales\", data=merge)\nplt.ylabel('sales', fontsize=12)\nplt.xlabel('City', fontsize=12)\nplt.title('Variation across cities', fontsize=15)\nplt.xticks(rotation='vertical')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.177357Z","iopub.status.idle":"2022-01-09T21:46:19.177719Z","shell.execute_reply.started":"2022-01-09T21:46:19.177502Z","shell.execute_reply":"2022-01-09T21:46:19.177544Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"store_items=pd.merge(sale_store_item_level,items,on='item_nbr')\nstore_items=pd.merge(store_items,stores,on='store_nbr')\nstore_items['item_sales']=store_items['item_sales']\n\n#item\n# top selling items by store type\ntop_items_by_type=store_items.groupby(['type','item_nbr'])['item_sales'].sum()\ntop_items_by_type=top_items_by_type.reset_index().sort_values(['type','item_sales'],ascending=[True,False])\n\n#get top 5\ntop_items_by_type=top_items_by_type.groupby(['type']).head(5)\n\n\n#class\n# top selling item class by store type\ntop_class_by_type=store_items.groupby(['type','class'])['item_sales'].sum()\ntop_class_by_type=top_class_by_type.reset_index().sort_values(['type','item_sales'],ascending=[True,False])\n\n#get top 5\ntop_class_by_type=top_class_by_type.groupby(['type']).head(5)\n\n\n#family\n# top selling item family by store type\ntop_family_by_type=store_items.groupby(['type','family'])['item_sales'].sum()\ntop_family_by_type=top_family_by_type.reset_index().sort_values(['type','item_sales'],ascending=[True,False])\n\n#get top 5\ntop_family_by_type=top_family_by_type.groupby(['type']).head(5)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.178701Z","iopub.status.idle":"2022-01-09T21:46:19.179031Z","shell.execute_reply.started":"2022-01-09T21:46:19.17885Z","shell.execute_reply":"2022-01-09T21:46:19.178874Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"top_family_by_type=store_items.groupby(['type','family'])['item_sales'].sum()\ntop_family_by_type=top_family_by_type.reset_index().sort_values(['type','item_sales'],ascending=[True,False])\nx=top_family_by_type.pivot(index='family',columns='type')\ncm = sns.light_palette(\"orange\", as_cmap=True)\nx = x.style.background_gradient(cmap=cm)\nx","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.180668Z","iopub.status.idle":"2022-01-09T21:46:19.180994Z","shell.execute_reply.started":"2022-01-09T21:46:19.180817Z","shell.execute_reply":"2022-01-09T21:46:19.18084Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The distribution of sale across the store types for the top 30 items have been shown below\nThe darker the color gradient the more the store type has contributed to the sale of items in that class\n","metadata":{}},{"cell_type":"code","source":"top_items_by_type=store_items.groupby(['type','item_nbr'])['item_sales'].sum()\ntop_items_by_type=top_items_by_type.reset_index().sort_values(['type','item_sales'],ascending=[True,False])\ntop_items_by_type=top_items_by_type.groupby(['item_nbr']).head(20)\n#print(top_items_by_type)\nx=top_items_by_type.pivot(index='item_nbr',columns='type')\nx['total']=x.sum(axis=1)\nx=x.sort_values('total',ascending=False)\ndel(x['total'])\nx=x.head(30)\ncm = sns.light_palette(\"green\", as_cmap=True)\nx = x.style.background_gradient(cmap=cm,axis=1)\nx","metadata":{"execution":{"iopub.status.busy":"2022-01-09T21:46:19.182437Z","iopub.status.idle":"2022-01-09T21:46:19.182789Z","shell.execute_reply.started":"2022-01-09T21:46:19.182615Z","shell.execute_reply":"2022-01-09T21:46:19.182638Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# feature engineering","metadata":{}},{"cell_type":"markdown","source":"**Here we analyze the data and select the features for our model to be trained on.**","metadata":{}},{"cell_type":"markdown","source":"**Train**\nid, date, store_nbr, item_nbr, unit_scale, on_promotion\n\n**Items**\nitem_nbr, family, class, perishable\n\n**Holidays_events**\ndate, type, locale, locale_name, description, transferred\n\n**Stores**\nstore_nbr, city, state, type, cluster\n\n**Oil**\ndate, dcoilwtico\n\n**Transactions**\ndate, store_nbr, transactions","metadata":{}},{"cell_type":"markdown","source":"**Selected features as inputs to the model**\n\ndate, holiday.type, holidaye.locale, holiday.locale_name, holiday_transfered, store_nbr, store.city, store.state, store.type, store.cluster, transactions, item_nbr, item.family, item.class, on_promotion, perishable, dcoilwtico.\n\n**Selected features as outputs of the model**\n\ntransactions per store, unit_sales per item","metadata":{}},{"cell_type":"markdown","source":"# DATA pipeline","metadata":{}},{"cell_type":"markdown","source":"### takes files in, outputs a complete dataframe","metadata":{}},{"cell_type":"code","source":"import datetime as dt\nfrom sklearn.base import BaseEstimator, TransformerMixin\n\nclass prepare_data(BaseEstimator, TransformerMixin):\n    def __init__(self):\n        print(\"prepare_data -> init\")\n    def fit(self, X, y=None):\n        return self\n    def transform(self, X, y=None):\n        train_stores = X[0].merge(X[1], right_on = 'store_nbr', left_on='store_nbr')\n        train_stores_oil = train_stores.merge(X[2], right_on='date', left_on='date')\n        train_stores_oil_items = train_stores_oil.merge(X[3], right_on = 'item_nbr', left_on = 'item_nbr')\n        train_stores_oil_items_transactions = train_stores_oil_items.merge(X[4], right_on = ['date', 'store_nbr'], left_on = ['date', 'store_nbr'])\n        train_stores_oil_items_transactions_hol = train_stores_oil_items_transactions.merge(X[5], right_on = 'date', left_on = 'date')\n        \n        data_df = train_stores_oil_items_transactions_hol.copy(deep = True)\n        \n        # change the bool to int\n        data_df['onpromotion'] = data_df['onpromotion'].astype(int)\n        data_df['transferred'] = data_df['transferred'].astype(int)\n\n        # change the names\n        data_df.rename(columns={'type_x': 'st_type', 'type_y': 'hol_type'}, inplace=True)\n\n        # drop the id\n        data_df.drop(['id'], axis=1, inplace=True)\n        \n        print(data_df.head())\n        \n        # handle date\n        data_df['date'] = pd.to_datetime(data_df['date'])\n        data_df['date'] = data_df['date'].map(dt.datetime.toordinal)\n                \n        return data_df","metadata":{"execution":{"iopub.status.busy":"2022-01-14T14:55:46.616968Z","iopub.execute_input":"2022-01-14T14:55:46.617986Z","iopub.status.idle":"2022-01-14T14:55:47.623201Z","shell.execute_reply.started":"2022-01-14T14:55:46.61794Z","shell.execute_reply":"2022-01-14T14:55:47.6222Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Custom transform for splitting the data","metadata":{}},{"cell_type":"code","source":"# split dataframe into numerical values, categorical values and date\nclass split_data(BaseEstimator, TransformerMixin):\n    def __init__(self):\n        print(\"split_data -> init\")\n    def fit(self, X, y=None):\n        return self\n    def transform(self, X, y=None):\n        # Get columns for each type         \n        df_ = X.drop(['date'], axis = 1)\n        cols = df_.columns\n        num_cols = df_._get_numeric_data().columns\n        cat_cols = list(set(cols) - set(num_cols))\n        \n        data_num_df = X[num_cols]\n        data_cat_df = X[cat_cols]\n        data_date_df = X['date']\n        \n        return data_num_df, data_cat_df, data_date_df","metadata":{"execution":{"iopub.status.busy":"2022-01-14T14:55:51.745988Z","iopub.execute_input":"2022-01-14T14:55:51.746302Z","iopub.status.idle":"2022-01-14T14:55:51.75371Z","shell.execute_reply.started":"2022-01-14T14:55:51.746272Z","shell.execute_reply":"2022-01-14T14:55:51.752999Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Custom transform\n\n1. Fill missing data in numerical attributes\n2. apply standard scaler to numerical attributes\n3. Convert categorical data into numerical","metadata":{}},{"cell_type":"code","source":"from sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.preprocessing import StandardScaler\n\nclass process_data(BaseEstimator, TransformerMixin):\n    def __init__(self):\n        print(\"process_data -> init\")\n    def fit(self, X, y=None):\n        return self\n    def transform(self, X, y=None):\n        ### numerical data\n        # impute nulls in numerical attributes\n        imputer = SimpleImputer(strategy=\"mean\", copy=\"true\")\n        num_imp = imputer.fit_transform(X[0])\n        data_num_df = pd.DataFrame(num_imp, columns=X[0].columns, index=X[0].index)\n        \n        # apply standard scaling\n        scaler = StandardScaler()\n        scaler.fit(data_num_df)\n        num_scaled = scaler.transform(data_num_df)\n        data_num_df = pd.DataFrame(num_scaled, columns=X[0].columns, index=X[0].index)\n        \n        ### categorical data\n        # one hot encoder\n        cat_encoder = OneHotEncoder(sparse=False)\n        data_cat_1hot = cat_encoder.fit_transform(X[1])\n        \n        # convert it to datafram with n*99 where n number of rows and 99 is no. of categories\n        data_cat_df = pd.DataFrame(data_cat_1hot, columns=cat_encoder.get_feature_names()) #, index=X[1].index)\n                \n        return data_num_df, data_cat_df, X[2]","metadata":{"execution":{"iopub.status.busy":"2022-01-14T14:55:55.020419Z","iopub.execute_input":"2022-01-14T14:55:55.021077Z","iopub.status.idle":"2022-01-14T14:55:55.281214Z","shell.execute_reply.started":"2022-01-14T14:55:55.021028Z","shell.execute_reply":"2022-01-14T14:55:55.280039Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### yet another transform","metadata":{}},{"cell_type":"code","source":"class join_df(BaseEstimator, TransformerMixin):\n    def __init__(self):\n        print(\"join_df -> init\")\n    def fit(self, X, y=None):\n        return self\n    def transform(self, X, y=None):\n        ### numerical data\n        data_df = X[0].join(X[1])\n        data_df = data_df.join(X[2])\n        \n        return data_df","metadata":{"execution":{"iopub.status.busy":"2022-01-14T14:55:58.525398Z","iopub.execute_input":"2022-01-14T14:55:58.525711Z","iopub.status.idle":"2022-01-14T14:55:58.531371Z","shell.execute_reply.started":"2022-01-14T14:55:58.525675Z","shell.execute_reply":"2022-01-14T14:55:58.530601Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler\n\npipe_processing = Pipeline([\n        ('prepare_data', prepare_data()),\n        ('split_data', split_data()),\n        ('process_data', process_data()),\n        ('join_data', join_df())\n    ])\n\n# our prepared data\ndata_df = pipe_processing.fit_transform([train_large, stores, oil, items, transactions, holiday_events])\n\n# split it according to our feature engineering\nX = data_df.drop(['unit_sales', 'transactions'], axis=1)\nY = data_df[['unit_sales', 'transactions']]","metadata":{"execution":{"iopub.status.busy":"2022-01-14T14:56:01.325628Z","iopub.execute_input":"2022-01-14T14:56:01.326086Z","iopub.status.idle":"2022-01-14T14:56:30.877176Z","shell.execute_reply.started":"2022-01-14T14:56:01.326051Z","shell.execute_reply":"2022-01-14T14:56:30.876142Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Generate test and training data\n","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nx_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=0.3, random_state=0)","metadata":{"execution":{"iopub.status.busy":"2022-01-14T14:56:39.031705Z","iopub.execute_input":"2022-01-14T14:56:39.032195Z","iopub.status.idle":"2022-01-14T14:56:40.732184Z","shell.execute_reply.started":"2022-01-14T14:56:39.032159Z","shell.execute_reply":"2022-01-14T14:56:40.731108Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Modelling and testing","metadata":{}},{"cell_type":"markdown","source":"### Generic function for modelling and testing","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression,SGDRegressor,ElasticNet,Ridge\nfrom sklearn.svm import SVC\nfrom sklearn import linear_model\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import mean_squared_error,mean_absolute_error\n\n\ndef checkModelPerformane(model):\n    model.fit(x_train.values, y_train.values)\n    \n    pred = model.predict(x_test.values)\n    \n    print(\"mean_squared_error: \",np.sqrt(mean_squared_error(y_test.values, pred))) \n    print(\"mean_absolute_error: \", np.sqrt(mean_absolute_error(y_test.values, pred)))","metadata":{"execution":{"iopub.status.busy":"2022-01-14T14:56:49.066344Z","iopub.execute_input":"2022-01-14T14:56:49.066676Z","iopub.status.idle":"2022-01-14T14:56:49.134593Z","shell.execute_reply.started":"2022-01-14T14:56:49.066641Z","shell.execute_reply":"2022-01-14T14:56:49.13359Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Linear regression","metadata":{}},{"cell_type":"code","source":"print(\"LinearRegression\")\ncheckModelPerformane(LinearRegression())","metadata":{"execution":{"iopub.status.busy":"2022-01-14T14:56:52.502808Z","iopub.execute_input":"2022-01-14T14:56:52.503454Z","iopub.status.idle":"2022-01-14T14:57:00.740279Z","shell.execute_reply.started":"2022-01-14T14:56:52.503398Z","shell.execute_reply":"2022-01-14T14:57:00.739222Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Lasso regression","metadata":{}},{"cell_type":"code","source":"print(\"lasso regression \")\ncheckModelPerformane(linear_model.Lasso(alpha=0.1))","metadata":{"execution":{"iopub.status.busy":"2022-01-14T14:57:03.544775Z","iopub.execute_input":"2022-01-14T14:57:03.545062Z","iopub.status.idle":"2022-01-14T14:57:06.401898Z","shell.execute_reply.started":"2022-01-14T14:57:03.545032Z","shell.execute_reply":"2022-01-14T14:57:06.397773Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ElasticNet regression","metadata":{}},{"cell_type":"code","source":"print(\"ElasticNet regression \")\ncheckModelPerformane(ElasticNet())","metadata":{"execution":{"iopub.status.busy":"2022-01-14T14:57:11.640405Z","iopub.execute_input":"2022-01-14T14:57:11.640718Z","iopub.status.idle":"2022-01-14T14:57:13.65368Z","shell.execute_reply.started":"2022-01-14T14:57:11.640677Z","shell.execute_reply":"2022-01-14T14:57:13.650641Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Ridge regression","metadata":{}},{"cell_type":"code","source":"print(\"Ridge regression \")\ncheckModelPerformane(Ridge(alpha=1.0))","metadata":{"execution":{"iopub.status.busy":"2022-01-14T14:57:16.541841Z","iopub.execute_input":"2022-01-14T14:57:16.542189Z","iopub.status.idle":"2022-01-14T14:57:18.039912Z","shell.execute_reply.started":"2022-01-14T14:57:16.542147Z","shell.execute_reply":"2022-01-14T14:57:18.038495Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Random forests","metadata":{}},{"cell_type":"code","source":"print(\"Random Forest\")\ncheckModelPerformane(RandomForestRegressor(random_state=42)) ","metadata":{"execution":{"iopub.status.busy":"2022-01-14T14:57:23.765511Z","iopub.execute_input":"2022-01-14T14:57:23.765828Z","iopub.status.idle":"2022-01-14T15:12:33.243859Z","shell.execute_reply.started":"2022-01-14T14:57:23.765792Z","shell.execute_reply":"2022-01-14T15:12:33.241081Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Random forest model has the lowest error, thus we are going to use it and fine tune it.","metadata":{}},{"cell_type":"markdown","source":"# Model fine tuning","metadata":{}},{"cell_type":"markdown","source":"### Grid search","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\n\nparam_grid = [\n    {'n_estimators': [3, 10, 30], 'max_features': [2, 4, 6, 8]},\n    {'bootstrap': [False], 'n_estimators': [3, 10], 'max_features': [2, 3, 4]},]\n\nforest_reg = RandomForestRegressor(random_state=42)\n \ngrid_search = GridSearchCV(forest_reg, param_grid, cv=5, scoring='neg_mean_squared_error', return_train_score=True)\ngrid_search.fit(x_train.values, y_train.values)","metadata":{"execution":{"iopub.status.busy":"2022-01-14T15:14:54.187511Z","iopub.execute_input":"2022-01-14T15:14:54.188813Z","iopub.status.idle":"2022-01-14T16:19:03.364353Z","shell.execute_reply.started":"2022-01-14T15:14:54.18875Z","shell.execute_reply":"2022-01-14T16:19:03.362878Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### find out the best parameters for our model","metadata":{}},{"cell_type":"code","source":"grid_search.best_params_","metadata":{"execution":{"iopub.status.busy":"2022-01-14T16:25:15.104903Z","iopub.execute_input":"2022-01-14T16:25:15.106073Z","iopub.status.idle":"2022-01-14T16:25:15.116144Z","shell.execute_reply.started":"2022-01-14T16:25:15.106026Z","shell.execute_reply":"2022-01-14T16:25:15.115214Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"grid_search.best_estimator_","metadata":{"execution":{"iopub.status.busy":"2022-01-14T16:25:18.9846Z","iopub.execute_input":"2022-01-14T16:25:18.984911Z","iopub.status.idle":"2022-01-14T16:25:18.992565Z","shell.execute_reply.started":"2022-01-14T16:25:18.984876Z","shell.execute_reply":"2022-01-14T16:25:18.99149Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Test model on test set","metadata":{}},{"cell_type":"code","source":"!head test.csv","metadata":{"execution":{"iopub.status.busy":"2022-01-14T15:14:11.517239Z","iopub.execute_input":"2022-01-14T15:14:11.517545Z","iopub.status.idle":"2022-01-14T15:14:12.453753Z","shell.execute_reply.started":"2022-01-14T15:14:11.517514Z","shell.execute_reply":"2022-01-14T15:14:12.452583Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_model = grid_search.best_estimator_\n\n# load and process data\ntest = pd.read_csv(\"../working/test.csv\", parse_dates=['date'])\n\npipe_processing2 = Pipeline([\n        ('split_data', split_data()),\n        ('process_data', process_data()),\n        ('join_data', join_df())\n    ])\n\ntest_df = pipe_processing2.fit_transform(test)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-01-14T16:30:53.543403Z","iopub.execute_input":"2022-01-14T16:30:53.543731Z","iopub.status.idle":"2022-01-14T16:31:01.972013Z","shell.execute_reply.started":"2022-01-14T16:30:53.543701Z","shell.execute_reply":"2022-01-14T16:31:01.971139Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df\n","metadata":{"execution":{"iopub.status.busy":"2022-01-14T16:31:04.551377Z","iopub.execute_input":"2022-01-14T16:31:04.551705Z","iopub.status.idle":"2022-01-14T16:31:04.570078Z","shell.execute_reply.started":"2022-01-14T16:31:04.551669Z","shell.execute_reply":"2022-01-14T16:31:04.56902Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# final_predictions = final_model.predict(test_x)","metadata":{"execution":{"iopub.status.busy":"2022-01-14T16:32:33.030161Z","iopub.execute_input":"2022-01-14T16:32:33.030472Z","iopub.status.idle":"2022-01-14T16:32:33.074074Z","shell.execute_reply.started":"2022-01-14T16:32:33.030439Z","shell.execute_reply":"2022-01-14T16:32:33.072111Z"},"trusted":true},"outputs":[],"execution_count":null}]}