{"nbformat": 4, "nbformat_minor": 1, "metadata": {"kernelspec": {"display_name": "Python 3", "name": "python3", "language": "python"}, "language_info": {"codemirror_mode": {"version": 3, "name": "ipython"}, "name": "python", "version": "3.6.3", "mimetype": "text/x-python", "nbconvert_exporter": "python", "file_extension": ".py", "pygments_lexer": "ipython3"}}, "cells": [{"metadata": {"_cell_guid": "e0a7d16d-ba20-4e15-919c-4f642c260afc", "_uuid": "ab8914489cb0bcfd7c243cf8760d4150dfd1d37e"}, "cell_type": "markdown", "source": ["<h1>Introduction</h1>\n", "the data is large so this kernal was run on only a small portion of the data, the data was broken down into store numbers and item numbers"]}, {"metadata": {"_cell_guid": "159b43cb-0557-4374-b5a3-9d16c58742f2", "_uuid": "41bd9b88a610be95a9da6884be31bd5c29159158"}, "cell_type": "markdown", "source": ["<h1>Correlating oil prices</h1>\n"]}, {"outputs": [], "source": ["import matplotlib.pyplot as plt\n", "from datetime import datetime\n", "import seaborn as sns\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "from sklearn.svm import SVR\n", "from sklearn.model_selection import train_test_split\n", "%matplotlib inline\n", "\n", "from subprocess import check_output\n", "\n", "!head -15000000 ../input/train.csv > small_test.csv\n", "\n", "dtype_holiday={'date':'O', 'type':'O', 'locale':'O', 'locale_name':'O', 'description':'O','transferred': 'bool'} \n", "holiday_df=pd.read_csv(\"../input/holidays_events.csv\",dtype=dtype_holiday)\n", "#print(holiday_df,\"holiday_df\")\n", "dfo_data={'date': 'O', 'dcoilwtico':'float64'}\n", "dfo=pd.read_csv(\"../input/oil.csv\",dtype=dfo_data)\n", "\n", "data_type={'date': 'O','id': 'int64','item_nbr':'int64','onpromotion':'float64','store_nbr': 'int64','unit_sales':'float64'}\n", "dft=pd.read_csv(\"small_test.csv\",dtype=data_type)\n", "\n", "def to_ordinal(x):\n", " d =datetime.strptime(x,'%Y-%m-%d')\n", " z=datetime.date(d)\n", " y=z.toordinal()\n", " return y\n", "dft[\"days\"]=dft[\"date\"].apply(to_ordinal)\n", "dft=dft.drop(\"date\",axis=1)\n", "dfo[\"days\"]=dfo[\"date\"].apply(to_ordinal)\n", "dfo=dfo.drop(\"date\",axis=1)\n", "holiday_df[\"days\"]=holiday_df[\"date\"].apply(to_ordinal)\n", "holiday_df=holiday_df.drop(\"date\",axis=1)\n", "name_set=set(holiday_df[\"type\"])\n", "name_list=list(name_set)\n", "holiday_code={}\n", "for x,y in zip(name_list,range(len(name_set))):\n", " holiday_code[y]=x\n", "holiday_df[\"type\"]=holiday_df[\"type\"].replace(name_list,range(len(name_set)))\n", "#print(holiday_df.head())    \n", "#merged=dft.merge(dfo,on=\"date\")\n", "merged=dft.merge(holiday_df,on=\"days\")\n", "merged=merged.merge(dfo,on=\"days\")\n", "#print(merged.head())\n", "merged=merged.drop([\"id\",\"locale\",\"description\",\"locale_name\",\"transferred\",\"onpromotion\",],axis=1)\n", "merged=merged.dropna()\n", "#print(merged.head())\n", "items2=set(merged[\"item_nbr\"])\n", "items2=list(items2)\n", "df_item_1=merged[merged.item_nbr==items2[0]]\n", "store_sets=set(df_item_1[\"store_nbr\"])\n", "store_sets=list(store_sets)\n", "\n", "def change_(x):\n", "    return holiday_code[x]\n", "df_item_1[\"type\"]=df_item_1[\"type\"].apply(change_)\n", "##store number 9, item number 638977\n", "df_store_and_item=df_item_1[df_item_1.store_nbr==store_sets[0]]\n", "#print(df_store_and_item.head())\n", "sns.pointplot(\"days\",\"unit_sales\",data=df_store_and_item,hue=\"type\")\n", "sns.pointplot(\"days\",\"dcoilwtico\",data=df_store_and_item,color=\"navy\")\n", "plt.show()\n", "##store number 10, item number 638977\n", "df_store_and_item1=df_item_1[df_item_1.store_nbr==store_sets[1]]\n", "#print(df_store_and_item1.head())\n", "sns.pointplot(\"days\",\"unit_sales\",data=df_store_and_item1,hue=\"type\")\n", "sns.pointplot(\"days\",\"dcoilwtico\",data=df_store_and_item1,color=\"navy\")\n", "plt.show()\n"], "execution_count": null, "cell_type": "code", "metadata": {"_cell_guid": "1447e7dc-85ce-4e75-bba2-bb8cffc67625", "collapsed": true, "_uuid": "0b8d2aa28ffa689385b2f0b37c54ba51bf160fd8"}}, {"metadata": {"_cell_guid": "903fa077-8797-446a-9f8d-8e10e72130bf", "_uuid": "f581ed32a84cfd2dedb3205441bbea5cb7320804"}, "cell_type": "markdown", "source": ["sales for item number 638977 at stores 9 and 10, are affected more by holidays rather than oil prices, first graph for store 9 and the second graph for store 10,   navy blue line oil prices\n"]}, {"metadata": {"_cell_guid": "cded40cd-95e4-46cb-97ce-092968218ca8", "_uuid": "eb108a038f8405b63d643d82ac9a3f3a3942a606"}, "cell_type": "markdown", "source": ["<h1>Using a SVR</h1>"]}, {"outputs": [], "source": ["# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "import matplotlib.pyplot as plt\n", "from datetime import datetime\n", "import seaborn as sns\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "from sklearn.svm import SVR\n", "from sklearn.model_selection import train_test_split\n", "%matplotlib inline\n", "\n", "dict_={'date':'O','item_nbr':'int64','onpromotion':'float64','store_nbr': 'int64'}\n", "from subprocess import check_output\n", "#print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "!head -15000000 ../input/train.csv > small_test.csv\n", "#!head -2 ../input/holidays_events.csv > holiday_test.csv\n", "dtype_holiday={'date':'O', 'type':'O', 'locale':'O', 'locale_name':'O', 'description':'O','transferred': 'bool'} \n", "holiday_df=pd.read_csv(\"../input/holidays_events.csv\",dtype=dtype_holiday)\n", "#print(holiday_df,\"holiday_df\")\n", "data_type={'date': 'O','id': 'int64','item_nbr':'int64','onpromotion':'float64','store_nbr': 'int64','unit_sales':'float64'}\n", "dft=pd.read_csv(\"small_test.csv\",dtype=data_type)\n", "\n", "def to_ordinal(x):\n", " d =datetime.strptime(x,'%Y-%m-%d')\n", " z=datetime.date(d)\n", " y=z.toordinal()\n", " return y\n", "dft[\"days\"]=dft[\"date\"].apply(to_ordinal)\n", "dft=dft.drop(\"date\",axis=1)\n", "name_set=set(holiday_df[\"type\"])\n", "holiday_df[\"days\"]=holiday_df[\"date\"].apply(to_ordinal)\n", "holiday_df=holiday_df.drop(\"date\",axis=1)\n", "name_list=list(name_set)\n", "holiday_code={}\n", "for x,y in zip(name_list,range(len(name_set))):\n", " holiday_code[y]=x\n", "holiday_df[\"type\"]=holiday_df[\"type\"].replace(name_list,range(len(name_set)))\n", "#print(holiday_df.head())    \n", "#merged=dft.merge(dfo,on=\"date\")\n", "merged=dft.merge(holiday_df,on=\"days\")\n", "#print(merged.head())\n", "merged=merged.drop([\"id\",\"locale\",\"description\",\"locale_name\",\"transferred\",\"onpromotion\",],axis=1)\n", "items2=set(merged[\"item_nbr\"])\n", "items2=list(items2)\n", "df_item_1=merged[merged.item_nbr==items2[0]]\n", "store_sets=set(df_item_1[\"store_nbr\"])\n", "store_sets=list(store_sets)\n", "df_store_and_item=df_item_1[df_item_1.store_nbr==store_sets[1]]\n", "#print(df_store_and_item.head())\n", "##store number 9, item number 638977\n", "#print(\"before fitting\") \n", "lables=df_store_and_item[\"unit_sales\"]\n", "#print(df_store_and_item.head())\n", "for_features=df_store_and_item.drop([\"unit_sales\",\"item_nbr\",\"store_nbr\",\"type\"],axis=1)\n", "for_features_train,for_features_test,lables_train,lables_test=train_test_split(for_features,lables,test_size=0.30,random_state=42)\n", "\n", "rbf_clf = SVR(kernel=\"rbf\")\n", "#linear_clf = SVR(kernel=\"linear\")\n", "#poly_clf = SVR(kernel=\"poly\")\n", "draw_rbf=rbf_clf.fit(for_features_train,lables_train).predict(for_features_test)\n", "#draw_linear=linear_clf.fit(for_features_train,lables_train).predict(for_features_test)\n", "df_store_and_item1=df_item_1[df_item_1.store_nbr==store_sets[1]]\n", "lables1=df_store_and_item1[\"unit_sales\"]\n", "for_features1=df_store_and_item1.drop([\"unit_sales\",\"item_nbr\",\"store_nbr\",\"type\"],axis=1)\n", "for_features_train1,for_features_test1,lables_train1,lables_test1=train_test_split(for_features1,lables1,test_size=0.30,random_state=42)\n", "draw1_rbf=rbf_clf.fit(for_features_train1,lables_train1).predict(for_features_test1)\n", "\n", "def change_(x):\n", "    return holiday_code[x]\n", "\n", "for_plotting=pd.DataFrame(columns=[\"unit_sales\",\"days\"])\n", "for_plotting[\"unit_sales\"]=lables_test\n", "for_plotting[\"days\"]=for_features_test[\"days\"]\n", "#sns.factorplot(\"ordinal\",\"unit_sales\",data=for_plotting,kind=\"point\")\n", "a=plt.subplot()\n", "a.scatter(for_plotting[\"days\"],for_plotting[\"unit_sales\"],color=\"black\")\n", "a.scatter(for_plotting[\"days\"],draw_rbf,color=\"orange\")\n", "a.set_title(\"store 9\")\n", "plt.xlabel('days')\n", "plt.ylabel('Unit_sales')\n", "plt.show()\n", "\n", "for_plotting1=pd.DataFrame(columns=[\"unit_sales\",\"days\"])\n", "for_plotting1[\"unit_sales\"]=lables_test1\n", "for_plotting1[\"days\"]=for_features_test1[\"days\"]\n", "b=plt.subplot()\n", "b.scatter(for_plotting1[\"days\"],for_plotting1[\"unit_sales\"],color=\"black\")\n", "b.scatter(for_plotting1[\"days\"],draw1_rbf,color=\"orange\")\n", "b.set_title(\"store 10\")\n", "plt.xlabel('days')\n", "plt.ylabel('Unit_sales')\n", "plt.show()\n"], "execution_count": null, "cell_type": "code", "metadata": {"_cell_guid": "e4cfb0b2-8b12-4f5f-be81-81fc0dfad455", "collapsed": true, "_uuid": "d3b11aaecbe4a24b9638b58c6311bd864a83d5b6"}}, {"metadata": {"_cell_guid": "38980b3d-b196-4032-ac79-b6c0b96ff758", "_uuid": "25ba5b5cd967e73737438a616546970fa94c860a"}, "cell_type": "markdown", "source": ["The data was split into 70% train and 30% test;\n", "This a graph for the test data, for item number 638977 in store number 9 and store number 10 using a SVR and an rbf kernal. store 9 should stock around 40 units of item 638977, and store 10 should stock around 20 units  oil prices not added for this model.\n", "\n", "<h1>Visualizing data differently</h1>\n", "the data was ploted based on a monthly basis below and trends appeard, sales were higer toward the beginning and they decreased towards the end of the month, using a gradient boost regressor seemed to give a pretty good estimate"]}, {"outputs": [], "source": ["from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "import matplotlib.pyplot as plt\n", "from datetime import datetime\n", "import seaborn as sns\n", "from sklearn.svm import SVR\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.ensemble import GradientBoostingRegressor\n", "\n", "\n", "\n", "from subprocess import check_output\n", "\n", "\n", "\n", "data_type={'date': 'O','id': 'int64','item_nbr':'int64','onpromotion':'float64','store_nbr': 'int64','unit_sales':'float64'}\n", "dft=pd.read_csv(\"../input/train.csv\",dtype=data_type,usecols=[1],parse_dates=[0])\n", "\n", "dft['year']=pd.DatetimeIndex(dft['date']).year\n", "dft['month']=pd.DatetimeIndex(dft['date']).month\n", "dft['day']=pd.DatetimeIndex(dft['date']).day\n", "dft['day']=dft['day'].astype(np.uint8)\n", "dft['month']=dft['month'].astype(np.uint8)\n", "dft['year']=dft['year'].astype(np.uint16)\n", "dft=dft.drop(\"date\",axis=1)\n", "\n", "\n", "dft1 = pd.read_csv(\"../input/train.csv\",dtype=data_type,usecols=[2,3,4])\n", "train=pd.concat([dft,dft1],axis=1)\n", "\n", "\n", "item_list=set(train[\"item_nbr\"])\n", "list_items=list(item_list)\n", "\n", "day_sort=set(train[\"day\"])\n", "list_day=list(day_sort)\n", "\n", "month_sort=set(train[\"month\"])\n", "list_month=list(month_sort)\n", "\n", "store_sort=set(train[\"store_nbr\"])\n", "list_store=list(store_sort)\n", "\n", "sort_by_item=train[train.item_nbr==list_items[0]]\n", "\n", "sort_by_month=sort_by_item[sort_by_item.month==list_month[2]]\n", "\n", "sort_by_store=sort_by_month[sort_by_month.store_nbr==list_store[9]]\n", "\n", "sns.pointplot(\"day\",\"unit_sales\",data=sort_by_store,size=4, aspect=2)\n", "plt.show()\n", "\n", "\n", "lables=sort_by_store[\"unit_sales\"]\n", "for_features=sort_by_store.drop([\"unit_sales\",\"item_nbr\",\"year\",\"month\"],axis=1)\n", "for_features_train,for_features_test,lables_train,lables_test=train_test_split(for_features,lables,test_size=0.30,random_state=42)\n", "\n", "clif2=GradientBoostingRegressor(n_estimators=100,learning_rate=1.0,loss='ls',random_state=0,max_depth=1).fit(for_features_train,lables_train)\n", "draw=clif2.predict(for_features_test)\n", "\n", "b=plt.subplot()\n", "b.scatter(for_features_test[\"day\"],lables_test,color=\"black\")\n", "b.scatter(for_features_test[\"day\"],draw,color=\"orange\")\n", "#b.set_title(\"store 10, item 638977\")\n", "plt.xlabel('days')\n", "plt.ylabel('Unit_sales')\n", "plt.show()"], "execution_count": null, "cell_type": "code", "metadata": {"_cell_guid": "55739315-39c0-48e9-aa23-d633686469a7", "collapsed": true, "_uuid": "c79af0261896ef2a90fbed1946e419626741c5b5"}}, {"metadata": {"_cell_guid": "8c2a42a5-ffba-49cf-bdb0-81f537c78be4", "_uuid": "b57fd09f1dc163ea50d4f5d98fab20fc5ae758c7"}, "cell_type": "markdown", "source": ["<h1>With a SVR</h1>"]}, {"outputs": [], "source": ["from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "import matplotlib.pyplot as plt\n", "from datetime import datetime\n", "import seaborn as sns\n", "from sklearn.svm import SVR\n", "from sklearn.model_selection import train_test_split\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "#print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output.\n", "\n", "#dtype_holiday={'date':'O', 'type':'O', 'locale':'O', 'locale_name':'O', 'description':'O','transferred': 'bool'} \n", "#holiday_df=pd.read_csv(\"../input/holidays_events.csv\",dtype=dtype_holiday)\n", "#print(holiday_df,\"holiday_df\")\n", "#dfo_data={'date': 'O', 'dcoilwtico':'float64'}\n", "#dfo=pd.read_csv(\"../input/oil.csv\",dtype=dfo_data)\n", "\n", "\n", "data_type={'date': 'O','id': 'int64','item_nbr':'int64','onpromotion':'float64','store_nbr': 'int64','unit_sales':'float64'}\n", "dft=pd.read_csv(\"../input/train.csv\",dtype=data_type,usecols=[1],parse_dates=[0])\n", "#print(dft.head())\n", "dft['year']=pd.DatetimeIndex(dft['date']).year\n", "dft['month']=pd.DatetimeIndex(dft['date']).month\n", "dft['day']=pd.DatetimeIndex(dft['date']).day\n", "dft['day']=dft['day'].astype(np.uint8)\n", "dft['month']=dft['month'].astype(np.uint8)\n", "dft['year']=dft['year'].astype(np.uint16)\n", "dft=dft.drop(\"date\",axis=1)\n", "#print(dft.head())\n", "\n", "dft1 = pd.read_csv(\"../input/train.csv\",dtype=data_type,usecols=[2,3,4])\n", "train=pd.concat([dft,dft1],axis=1)\n", "#print(dft1.head())\n", "\n", "item_list=set(train[\"item_nbr\"])\n", "list_items=list(item_list)\n", "\n", "day_sort=set(train[\"day\"])\n", "list_day=list(day_sort)\n", "\n", "month_sort=set(train[\"month\"])\n", "list_month=list(month_sort)\n", "\n", "store_sort=set(train[\"store_nbr\"])\n", "list_store=list(store_sort)\n", "#print(list_store)\n", "sort_by_item=train[train.item_nbr==list_items[0]]\n", "#print(sort_by_item.head())\n", "sort_by_month=sort_by_item[sort_by_item.month==list_month[0]]\n", "#print(sort_by_month)\n", "#sort_by_day=sort_by_month[sort_by_item.day==list_day[3]]\n", "#print(sort_by_day.head())\n", "sort_by_store=sort_by_month[sort_by_month.store_nbr==list_store[9]]\n", "#print(sort_by_store.head())\n", "sns.pointplot(\"day\",\"unit_sales\",data=sort_by_store)\n", "plt.show()\n", "\n", "#sort_by_store=sort_by_store.drop(\"year\",axis=1)\n", "lables=sort_by_store[\"unit_sales\"]\n", "for_features=sort_by_store.drop([\"unit_sales\",\"item_nbr\",\"year\"],axis=1)\n", "for_features_train,for_features_test,lables_train,lables_test=train_test_split(for_features,lables,test_size=0.30,random_state=42)\n", "rbf_clf = SVR(kernel=\"rbf\")\n", "draw_rbf=rbf_clf.fit(for_features_train,lables_train).predict(for_features_test)\n", "\n", "b=plt.subplot()\n", "b.scatter(for_features_test[\"day\"],lables_test,color=\"black\")\n", "b.scatter(for_features_test[\"day\"],draw_rbf,color=\"orange\")\n", "#b.set_title(\"store 10, item 638977\")\n", "plt.xlabel('days')\n", "plt.ylabel('Unit_sales')\n", "plt.show()"], "execution_count": null, "cell_type": "code", "metadata": {"_cell_guid": "08e7ac3c-fce0-4635-a8f0-c31fc12816b2", "collapsed": true, "_uuid": "cc003cd2a8747b0fe4b75b08504efed41049baca"}}]}