{"cells": [{"metadata": {"collapsed": true, "_uuid": "cc5430f7bf5ead9a8fc527e9fbebf4fabf8bc331", "_cell_guid": "dba74da2-aa42-4b8b-a157-0252673387c3"}, "source": ["### <font color='salmon'>  Corporaci\u00f3n Favorita Grocery Sales Forecasting\n"], "cell_type": "markdown"}, {"metadata": {"_uuid": "29a22939eb8f23de5fe6818c1ec414e1c5287a1e", "_cell_guid": "16fdbfde-927a-48cd-a69f-0fa8a621fee0"}, "source": ["### <font color='salmon'>Importing all the libraries that we will need</font>"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "bfbe11ecabe55ee24c5799f4cc5caba38c5dcf8e", "_cell_guid": "50ab73ad-9ff8-45c8-9f60-56346bc58de5"}, "source": ["# import necessary modules\n", "import numpy as np\n", "import pandas as pd\n", "import random\n", "import seaborn as sns\n", "import matplotlib.pyplot as plt\n", "import gc\n", "\n", "import seaborn as sns\n", "sns.set(style = 'whitegrid', color_codes = True)\n", "%matplotlib inline\n", "\n", "#For statistical tests\n", "import scipy.stats as st\n", "\n", "#For formula notation (similar to R)\n", "import statsmodels.formula.api as smf\n", "\n", "from sklearn.ensemble import RandomForestRegressor\n", "from sklearn.cross_validation import train_test_split\n", "import xgboost as xgb\n", "import operator"], "cell_type": "code"}, {"metadata": {"_uuid": "400bc03d9ee011991e35328a4e6f1d55d99559e7", "_cell_guid": "2e3ec664-e555-4636-8031-f7a10f55e034"}, "source": ["### <font color='salmon'>Read the Data</font>"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "5e7160e4b259c89977e9101c7a2362a58e1eccab", "_cell_guid": "030b7af3-e6ce-44a0-a317-12db5558e243"}, "source": ["df_train = pd.read_csv(\"../input/corporitasampled-train-data/train_rd.csv\")"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "cbaf21b04157d216228f664435c4e1274f8982f8", "_cell_guid": "0c2925e3-22db-4c7a-8f7b-8bcaa9a703fe"}, "source": ["df_train.head()"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "76c658714c1f3af4d8357c0a3deea4927acfe38f", "_cell_guid": "0debcbc5-010e-4fc4-875e-8b6a5d772cf8"}, "source": ["print(\"we have taken \",len(df_train), \"rows\")"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "84c865fda5090135ee91912a5042ab6871d88636", "_cell_guid": "8c43c67a-a9a0-4a84-9d6f-5719cd266d36"}, "source": ["Input_Path = '../input/favorita-grocery-sales-forecasting/'"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "bc10ee66aa7c855ca80a049ba8afaed895107b93", "_cell_guid": "749e9a44-8f19-4601-9015-7bfce8b1b7ec"}, "source": ["test  = pd.read_csv(\"../input/favorita-grocery-sales-forecasting/test.csv\")\n", "testg  = pd.read_csv(\"../input/favorita-grocery-sales-forecasting/test.csv\")\n", "store = pd.read_csv(\"../input/favorita-grocery-sales-forecasting/stores.csv\")\n", "holiday = pd.read_csv(\"../input/favorita-grocery-sales-forecasting/holidays_events.csv\")\n", "item = pd.read_csv(\"../input/favorita-grocery-sales-forecasting/items.csv\")\n", "oil = pd.read_csv(\"../input/favorita-grocery-sales-forecasting/oil.csv\")\n", "trans = pd.read_csv(\"../input/favorita-grocery-sales-forecasting/transactions.csv\")"], "cell_type": "code"}, {"metadata": {"_uuid": "24874af2407d8ae978f4c00590e5cd07fe25d996", "_cell_guid": "6170d206-a386-4b1d-bfe4-de049390da6c"}, "source": ["### <font color='salmon'>Getting familiar with the data</font>"], "cell_type": "markdown"}, {"metadata": {"_uuid": "bef4b5b49ca5f3880e525d11fe7bdd01e648bb74", "_cell_guid": "f49e17d9-0718-478e-95ad-1a8b71f0cc3d"}, "source": ["### <font color='salmon'>train data</font>\n", "\n", "* Training data, which includes the target unit_sales by date, store_nbr, and item_nbr and a unique id to label rows.\n", "* The target unit_sales can be integer (e.g., a bag of chips) or float (e.g., 1.5 kg of cheese).\n", "* Negative values of unit_sales represent returns of that particular item.\n", "* The onpromotion column tells whether that item_nbr was on promotion for a specified date and store_nbr.\n", "* Approximately 16% of the onpromotion values in this file are NaN."], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "fc76e153f599791ad9554add9f1653642f3bd8fd", "_cell_guid": "6722fc38-107e-466e-8dbd-b13302c6b878"}, "source": ["df_train.head()"], "cell_type": "code"}, {"metadata": {"_uuid": "fcb5eb13b8da5607df41acb37492037406487748", "_cell_guid": "708ad8bd-ab08-47a8-affa-997c7434eb33"}, "source": ["### <font color='salmon'>Items data</font>\n", "\n", "* Item metadata, including family, class, and perishable.\n", "* NOTE: Items marked as perishable have a score weight of 1.25; otherwise, the weight is 1.0"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "c3ab44e14833839a17c92b00be9e54ad85b80f97", "_cell_guid": "7606b3b3-2317-4a79-bf48-7dcf18dec13b"}, "source": ["item.head()"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "fa37c8cebb0d2ac4aad0041c01766d5066155889", "_cell_guid": "cdcb5ada-10c7-44a9-8fd8-2e555655724d"}, "source": ["print(\"There are\",len(item['family'].unique()),\"families of products or items\")"], "cell_type": "code"}, {"metadata": {"_uuid": "58a97fbe7f20cd42481c14ca9ada615d21eb3e75", "_cell_guid": "929ea90e-c0ff-4fff-b4b6-3a09c968019d"}, "source": ["### <font color='salmon'>Oil data</font>\n", "\n", "Daily oil price. Includes values during both the train and test data timeframe. (Ecuador is an oil-dependent country and it's economical health is highly vulnerable to shocks in oil prices.)"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "3ea06e36cc8877b6446ecfb8baa39e05ff6f0742", "_cell_guid": "a067487c-b803-4f56-9090-62ffe1e2cad3"}, "source": ["oil.head()"], "cell_type": "code"}, {"metadata": {"_uuid": "2c3a87e85494b63c7611e80fa80b8cfce3600b03", "_cell_guid": "67497b91-39a2-4790-a6b6-bac70d90b86c"}, "source": ["### <font color='salmon'>Transaction data</font>\n", "* The count of sales transactions for each date, store_nbr combination. \n", "* Only included for the training data timeframe."], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "2481e41ccb88c96d219073b754e7f222e2f209b1", "_cell_guid": "4c32d43e-79d0-4cb6-be5c-f77d6944fb38"}, "source": ["trans.head()"], "cell_type": "code"}, {"metadata": {"_uuid": "5959c32219abb7ad21602bc2f101effd5e8a2e7d", "_cell_guid": "90c3dc90-c85a-4883-803e-eaec6d8df9d0"}, "source": ["### <font color='salmon'>Holiday data</font>\n", "\n", "* Holidays and Events, with metadata\n", "* A holiday that is transferred officially falls on that calendar day, but was moved to another date by the government. \n", "A transferred day is more like a normal day than a holiday. To find the day that it was actually celebrated, look for the corresponding row where type is Transfer. For example, the holiday Independencia de Guayaquil was transferred from 2012-10-09 to 2012-10-12, which means it was celebrated on 2012-10-12. Days that are type Bridge are extra days that are added to a holiday (e.g., to extend the break across a long weekend). These are frequently made up by the type Work Day which is a day not normally scheduled for work (e.g., Saturday) that is meant to payback the Bridge."], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "99bbc7b5eaef8e211e258ef63e6248d03cb7573a", "_cell_guid": "77eead7d-67fc-4884-b380-7b1114030740"}, "source": ["holiday.head()"], "cell_type": "code"}, {"metadata": {"_uuid": "7fdb472c2ab9f19782df875c7aeafc87c6e28db3", "_cell_guid": "e73d4c3c-5728-4eb5-b28a-04e31f7fecb7"}, "source": ["### <font color='salmon'>Store data</font>\n", "* Store metadata, including city, state, type, and cluster.\n", "* cluster is a grouping of similar stores."], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "d9f8b5c0be5544b514f4614559af303ea097c196", "_cell_guid": "e0c9697f-4d82-422a-b75a-4e3705180f4f"}, "source": ["store.head()"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "8ffecb50b9d9326a8ec37e6c18ce55c673eb29e7", "_cell_guid": "8301b723-4ac3-431c-82e8-6a98bd5cc5ae"}, "source": ["print(\"There are\",len(store['type'].unique()),\"type of stores\")"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "aab810d942be2e940ce0a6907ad4a0e19b627198", "_cell_guid": "300918ea-2d14-45a7-a940-ac24d3f0dd0f"}, "source": ["print(\"Stores are in \",len(store['city'].unique()),\"cities in \", len(store['state'].unique()),\"states\")"], "cell_type": "code"}, {"metadata": {"_uuid": "1fe4fc988e054bf43aded26c99149be223fb26bb", "_cell_guid": "fdc49958-fa54-40e7-a719-3d9b84db8f24"}, "source": ["### <font color='salmon'>Test data</font>"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "02ae8aa7b2b406c4aa480071f587a5189f673bc8", "_cell_guid": "fcff027e-3d3f-4fae-aaef-a902be482113"}, "source": ["test.head()"], "cell_type": "code"}, {"metadata": {"_uuid": "df2defbc2f8625d716a9af4c45bbf748ca18f295", "_cell_guid": "13757936-a567-423d-8ea0-bddc120d4554"}, "source": ["### <font color='salmon'>Join Data</font>\n", "\n", "* Join stores and items data to Trian data with respect to Store number and Item number\n", "* Join holiday and oil data to Train with respect to Date"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "7277f8d5eeb5f701df5360bbe27b97c6a499e48e", "_cell_guid": "0f064800-67b5-4e4c-93ff-8485c60a55c5"}, "source": ["train = pd.merge(df_train, store, on= \"store_nbr\")\n", "train = pd.merge(train, item, on= \"item_nbr\")\n", "train = pd.merge(train, holiday, on=\"date\")\n", "train = pd.merge(train, oil, on =\"date\")"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "d984c95cb23d1d02e8c02f3478fa5f01a710bb89", "_cell_guid": "4da3243a-046b-4021-9f29-95603029f582"}, "source": ["train.head()"], "cell_type": "code"}, {"metadata": {"_uuid": "5c31499f1c1f9158812dd801b65ebf3f951cdb06", "_cell_guid": "fb0af1d1-b47e-441c-88d4-b8a83921722a"}, "source": ["Join Train and item data"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "a0c52e03d427dffeb4f4fb648f06df0d714e11ac", "_cell_guid": "4bb450db-f98f-4c4f-9ade-6b2cc0287da7"}, "source": ["train_items = pd.merge(df_train, item, how='inner')\n", "train_items1 = pd.merge(df_train, item, how='inner')\n", "train_items2 = pd.merge(df_train, item, how='inner')"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"scrolled": true, "collapsed": true, "_uuid": "0a247e28bef5aadf0d33e47460e012290aeaf679", "_cell_guid": "cfb7f1a2-b5bf-4e89-9c73-00f11e06c210"}, "source": ["train_items.head()"], "cell_type": "code"}, {"metadata": {"_uuid": "c924be1b9e43adced461a23705e5ccad8b5d4cd5", "_cell_guid": "d697b956-1cd7-4b95-96dc-3dae17370f64"}, "source": ["### <font color='salmon'> Data Pre Processing</font>"], "cell_type": "markdown"}, {"metadata": {"_uuid": "a7f1c1803e346eb4980030e4ddaf62963953ea01", "_cell_guid": "d76220ae-bd75-4924-af20-3698ef65d45b"}, "source": ["### <font color='salmon'> Checking for missing Data</font>"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "22a73b3c984c19387735e66f15d116e130efd4d0", "_cell_guid": "79e7cc4d-7f85-4870-8e27-041d2f8829eb"}, "source": ["oil_nan = (oil.isnull().sum() / oil.shape[0]) * 100\n", "oil_nan"], "cell_type": "code"}, {"metadata": {"_uuid": "6937c2d5a3744b1dcddd422cc085dcfaa44dacb4", "_cell_guid": "1cbcee94-f312-4b5d-934a-d3aefb4cd3cf"}, "source": ["Only 3.5% missing data on oil price"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "27dae7f288e41b49068b9c2d272d39c0d1f8c233", "_cell_guid": "6cd6861a-c764-48e1-9c0c-3eea79cbaa7b"}, "source": ["store_nan = (store.isnull().sum() / store.shape[0]) * 100\n", "store_nan"], "cell_type": "code"}, {"metadata": {"_uuid": "bcbda6b81393d4514a39b060c42cb3bf76c787ca", "_cell_guid": "1bc1a996-dcd6-4aa6-a34a-ef2c00daba71"}, "source": ["No missing store data"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "0b586c60f67a51233256ebf16f0be0754910ca91", "_cell_guid": "2f985fb8-ca8c-4e05-b933-6784c215ef62"}, "source": ["item_nan = (item.isnull().sum() / item.shape[0]) * 100\n", "item_nan"], "cell_type": "code"}, {"metadata": {"_uuid": "85bc0da3865456f10def683628d6b0370b2cd719", "_cell_guid": "d115deb9-f9e7-4cbc-90f8-217850cff708"}, "source": ["No missing item data"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "c0757662cfc6b5b2eb6da6372d4d87cf97af0a3c", "_cell_guid": "9a0fcf8b-bc74-4fbd-9545-d795ac9e156b"}, "source": ["df_train_nan = (df_train.isnull().sum() / df_train.shape[0]) * 100\n", "df_train_nan"], "cell_type": "code"}, {"metadata": {"_uuid": "73f893bec09071ca134303d5f87a0b85092eec18", "_cell_guid": "26170bb1-b946-4c29-926a-ce48ea29e32d"}, "source": ["17.3% missing data on onpromotion"], "cell_type": "markdown"}, {"metadata": {"_uuid": "1f3a5a7d0cc34ac4d00a104f82ebe94eb133aafa", "_cell_guid": "adc2cb09-380a-471a-bbf3-696c1615d511"}, "source": ["On promotion NAN values are UNKNOWN - if item is on promotion \n", "\n", "So replacing Nan of \"on promotion\" with 2 to indicate the items have unknown status on promotion"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "919afaa3df5a35be08cb7aab27a68cb68752bfe5", "_cell_guid": "926ff836-ee8c-44b0-ae73-1f5aeceeafd1"}, "source": ["train['onpromotion'] = train['onpromotion'].fillna(2)\n", "train['onpromotion'] = train['onpromotion'].replace(True,1)\n", "train['onpromotion'] = train['onpromotion'].replace(False,0)"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "2337c99bd88fc415a2c77dedd68be35edc083627", "_cell_guid": "2a11f5b5-8848-454b-a725-a66b45ded5a5"}, "source": ["(train['onpromotion'].unique())"], "cell_type": "code"}, {"metadata": {"_uuid": "8dd98dd204e89771bb0750687f8bc38284a254df", "_cell_guid": "b312f54f-bd16-4acc-8d2a-5438303c98e8"}, "source": ["Unknown oil price - putting 0 for now"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "7a12d553257202a950dfd04a99f1a3e86888d0a9", "_cell_guid": "46cf963f-250e-45c2-a073-64d452c401a7"}, "source": ["train['dcoilwtico'] = train['dcoilwtico'].fillna(0)"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "e1a04af6bb03be9d8de9acb8398a868e7b0021b6", "_cell_guid": "14f4724f-dea8-4199-bc86-e01827fe454b"}, "source": ["train['Year']  = train['date'].apply(lambda x: int(str(x)[:4]))\n", "train['Month'] = train['date'].apply(lambda x: int(str(x)[5:7]))\n", "train['date']  = train['date'].apply(lambda x: (str(x)[8:]))\n", "\n", "\n", "test['Year']  = test['date'].apply(lambda x: int(str(x)[:4]))\n", "test['Month'] = test['date'].apply(lambda x: int(str(x)[5:7]))\n", "test['date']  = test['date'].apply(lambda x: (str(x)[8:]))\n", "\n", "train.head()\n"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "cac8a08018342c95617f23f9b49271175eec9633", "_cell_guid": "0aabe4c1-5630-4d16-8249-0af9c0247bfd"}, "source": ["train_items1['date'] = pd.to_datetime(train_items1['date'], format='%Y-%m-%d')\n", "train_items1['day_item_purchased'] = train_items1['date'].dt.day\n", "train_items1['month_item_purchased'] =train_items1['date'].dt.month\n", "train_items1['quarter_item_purchased'] = train_items1['date'].dt.quarter\n", "train_items1['year_item_purchased'] = train_items1['date'].dt.year\n", "train_items1.drop('date', axis=1, inplace=True)\n", "\n", "train_items2['date'] = pd.to_datetime(train_items2['date'], format='%Y-%m-%d')\n", "train_items2['day_item_purchased'] = train_items2['date'].dt.day\n", "train_items2['month_item_purchased'] =train_items2['date'].dt.month\n", "train_items2['quarter_item_purchased'] = train_items2['date'].dt.quarter\n", "train_items2['year_item_purchased'] = train_items2['date'].dt.year\n", "train_items2.drop('date', axis=1, inplace=True)"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "0a2d9961cb4efef1a09488915e4f3675e59c7e3d", "_cell_guid": "3bc4fb44-e252-46c8-ab05-f3b9ca248583"}, "source": ["#train_items['Year']  = train_items['date'].apply(lambda x: int(str(x)[:4]))\n", "#train_items['Month'] = train_items['date'].apply(lambda x: int(str(x)[5:7]))\n", "#train_items['date']  = train_items['date'].apply(lambda x: (str(x)[8:]))"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "0c7b9da09131dfb589c71283353c5ccc6c9fc500", "_cell_guid": "1230e557-c78e-4be2-b868-1a0ccceab8eb"}, "source": ["train_items1.loc[(train_items1.unit_sales<0),'unit_sales'] = 1 \n", "train_items1['unit_sales'] =  train_items1['unit_sales'].apply(pd.np.log1p) \n", "\n", "train_items1['family'] = train_items1['family'].astype('category')\n", "train_items1['onpromotion'] = train_items1['onpromotion'].astype('category')\n", "train_items1['perishable'] = train_items1['perishable'].astype('category')\n", "cat_columns = train_items1.select_dtypes(['category']).columns\n", "train_items1[cat_columns] = train_items1[cat_columns].apply(lambda x: x.cat.codes)\n", "\n", "train_items2.loc[(train_items2.unit_sales<0),'unit_sales'] = 1 \n", "train_items2['unit_sales'] =  train_items2['unit_sales'].apply(pd.np.log1p) \n", "\n", "train_items2['family'] = train_items2['family'].astype('category')\n", "train_items2['onpromotion'] = train_items2['onpromotion'].astype('category')\n", "train_items2['perishable'] = train_items2['perishable'].astype('category')\n", "cat_columns = train_items2.select_dtypes(['category']).columns\n", "train_items2[cat_columns] = train_items2[cat_columns].apply(lambda x: x.cat.codes)"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "5e05d7b89400517144030538a995b8f33e7e2090", "_cell_guid": "fe4baecd-ac51-4737-b563-2d40f40ac20a"}, "source": ["train_items1.head()"], "cell_type": "code"}, {"metadata": {"_uuid": "dc3ca5bc7c130999d101ff9dbe9c1bba3c529800", "_cell_guid": "83ea90b1-e311-48fe-8429-e47e10fa54cb"}, "source": ["### <font color='salmon'> EDA </font>"], "cell_type": "markdown"}, {"metadata": {"_uuid": "da47d47ec5913dcc55c649f0f9882ed8ed610bbd", "_cell_guid": "653970fd-cc85-4fbd-8a7d-ff01d4c8a124"}, "source": ["We have taken a sample of train data to plot the graphs - the entire data is taking too much time"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "9673553de973b883ae553d55ca05da51f3d0d0a0", "_cell_guid": "90ba5338-ac11-4832-be3f-c83a368aeeea"}, "source": ["strain = train.sample(frac=0.01,replace=True)"], "cell_type": "code"}, {"metadata": {"_uuid": "e041f83d4b9cff97f35e6d1461b8d2934c35c569", "_cell_guid": "f2dfc5a9-8fd4-4723-8a43-449f7888a742"}, "source": ["### <font color='salmon'> Plotting Sales with promotion </font>"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "a72cd2c951c09bab6389a445bf1f427d7e1af32c", "_cell_guid": "e8f2a4d4-b735-43ac-86fc-4911f6d6baf3"}, "source": ["fig, (axis1) = plt.subplots(1,1,figsize=(30,4))\n", "sns.barplot(x='onpromotion', y='unit_sales', data=strain, ax=axis1)"], "cell_type": "code"}, {"metadata": {"_uuid": "cf6bd9a10d3b47435423ed938ccbe8c6e4383e90", "_cell_guid": "8eb86475-9b03-4e2a-a54f-280d4d8b692a"}, "source": ["### <font color='salmon'> Plotting Sales per Item Family </font>"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "b81acacc21b4807ffc3f526463e718fa277e6d42", "_cell_guid": "21756532-7a45-49aa-8ead-88b4eb806dfc"}, "source": ["fig, (axis1) = plt.subplots(1,1,figsize=(30,4))\n", "sns.barplot(x='family', y='unit_sales', data=strain, ax=axis1)"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "220a3f419be78833941a9f32151685a268f6d8aa", "_cell_guid": "7987fcf8-3a59-40d5-8b6d-7e5a3132a2b8"}, "source": ["fig, (axis1) = plt.subplots(1,1,figsize=(30,4))\n", "sns.countplot(x=strain['family'], data=strain, ax=axis1)"], "cell_type": "code"}, {"metadata": {"_uuid": "e7f176dc7878fca09a961c5b8f00ed4afe1bfd90", "_cell_guid": "f0802964-bf18-4896-b4d2-46db94586762"}, "source": ["### <font color='salmon'> Plotting Sales per Store Type </font>"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "109503199a4da7aea1b9a563b793fb91087d3229", "_cell_guid": "aa5f3c79-aef6-46f2-b91a-e505fef670ba"}, "source": ["fig, (axis1) = plt.subplots(1,1,figsize=(15,4))\n", "sns.barplot(x='type_x', y='unit_sales', data=strain, ax=axis1)"], "cell_type": "code"}, {"metadata": {"collapsed": true, "_uuid": "95da7b9f859f8a1288b99e11645f9cc950b0a259", "_cell_guid": "2cce4ec0-17c3-46c5-999b-d72ea00bbc57"}, "source": ["Store type \"A\" is highest selling"], "cell_type": "markdown"}, {"metadata": {"_uuid": "bdd187a87ecb548531979db985ef451597a1b818", "_cell_guid": "6d0d3fc9-ca57-4740-9ed5-ffe18b3e0b58"}, "source": ["### <font color='salmon'> Plotting Stores in Cities and states </font>"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "6fa04fb0fe4a4a7f38fff00faf6c73a0ef1bf366", "_cell_guid": "4bbbd689-02e0-4253-9d12-a93aa1810005"}, "source": ["fig, (axis1) = plt.subplots(1,1,figsize=(30,4))\n", "sns.countplot(x=store['city'], data=store, ax=axis1)"], "cell_type": "code"}, {"metadata": {"_uuid": "e355666e858617e0be4d93b7fe20ee6e03f11acc", "_cell_guid": "f07d3148-eff8-49dc-befa-d66ef54234fa"}, "source": ["Quito and Guayaquil have the most number of stores"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "e9c4550b13b00dcd20932c6a4d3248729800fed8", "_cell_guid": "d1ec71f5-12ce-401b-871a-b12c36f73066"}, "source": ["fig, (axis1) = plt.subplots(1,1,figsize=(30,4))\n", "sns.countplot(x=store['state'], data=store, ax=axis1)"], "cell_type": "code"}, {"metadata": {"_uuid": "7e56fc2d11f0724c8441332fc549667072db1d23", "_cell_guid": "b0e8d6c6-79dd-4cfc-99c6-979aa3ed3b75"}, "source": ["Pichincha and Guayas have highest number of stores"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "05207656ff76e599de42909f29a4e04ac4989eab", "_cell_guid": "96e6f3cc-ea7b-41e8-ae4b-364f6cd1e57a"}, "source": ["fig, (axis1) = plt.subplots(1,1,figsize=(30,4))\n", "sns.countplot(x='cluster', data=store, ax=axis1)"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "ca71aaf0f827b01db0c2dd3c2d39cdea88e98826", "_cell_guid": "e32554f6-9b7b-4f67-b69a-87bb6a1a7353"}, "source": ["g = sns.FacetGrid(train, col='cluster', hue='cluster', size=4)\n", "g.map(sns.barplot, 'type_x', 'unit_sales');"], "cell_type": "code"}, {"metadata": {"_uuid": "3d1e770ec63d0f462ce68f6f2bd5e0e22b6160ee", "_cell_guid": "0d7b1a5b-7c61-4e19-936c-8705ac967ad9"}, "source": ["#### Plotting Oil Price"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "d601648082b6a91160df4f83f3ef0cd225a68e72", "_cell_guid": "2d71741d-4199-4ee8-9356-9f62b41024b4"}, "source": ["fig, (axis1) = plt.subplots(1,1,sharex=True,figsize=(15,8))\n", "\n", "ax1 = oil.plot(legend=True,ax=axis1,marker='o',title=\"Oil Price\")\n"], "cell_type": "code"}, {"metadata": {"_uuid": "a62b0e9a65abb2915547db63f7d693384610724d", "_cell_guid": "ae4d4481-8b92-48e9-915b-4a525855bc06"}, "source": ["### Plotting Sales with date"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "93a20a7c6eb50f1be61034c9cec4b9d8b0670071", "_cell_guid": "50d89df2-8f73-40e6-93bb-7e76cb8ba4af"}, "source": ["average_sales = train.groupby('date')[\"unit_sales\"].mean()\n", "average_promo = train.groupby('date')[\"onpromotion\"].mean()\n", "\n", "fig, (axis1, axis2) = plt.subplots(2,1,figsize=(15,4))\n", "\n", "ax1 = average_sales.plot(legend=True,ax=axis1,marker='o',title=\"Average Sales\")\n", "ax2 = average_promo.plot(legend=True,ax=axis2,marker='o',rot=90,colormap=\"summer\",title=\"Average Promo\")"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "e9794e7ed0799c9ee39959b7ba1fa85dbf996dea", "_cell_guid": "0aa5e5c8-8c2c-44f7-845e-aadb92bad9da"}, "source": ["train.plot(kind='scatter',x='store_nbr',y='unit_sales',figsize=(15,4))"], "cell_type": "code"}, {"metadata": {"_uuid": "fd8ba6b13aa34f01f6dc4e5d25bca8e0dfac1243", "_cell_guid": "3a40efa2-c1fc-488a-8111-09704f908537"}, "source": ["Store number 44 / 45 - maximum sales"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "89ea300372ec5224f3b14a12333264b2dd375a9c", "_cell_guid": "64a9e17a-3ab4-4cac-a3a8-582fc72b6d2e"}, "source": ["train.plot(kind='scatter',x='item_nbr',y='unit_sales',figsize=(15,4))"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "1fe2fd7718d502788199475f1163f2535d790cda", "_cell_guid": "8061d4d4-6bcc-487d-bd70-f0aaec26bd04"}, "source": ["store_number = train.groupby('store_nbr')[\"unit_sales\"].mean()\n", "item_number = train.groupby('item_nbr')[\"unit_sales\"].mean()\n", "\n", "fig, (axis1, axis2) = plt.subplots(2,1,figsize=(30,4))\n", "\n", "ax1 = store_number.plot(legend=True,ax=axis1,marker='o',title=\"Sales with store\")\n", "ax2 = item_number.plot(legend=True,ax=axis2,marker='o',rot=90,colormap=\"summer\",title=\"Sales with item\")"], "cell_type": "code"}, {"metadata": {"_uuid": "5793087b2287fb15c93722789775ea77754769ef", "_cell_guid": "e146fde4-aae0-4a79-aefe-21f7a6e864c9"}, "source": ["Store 45 - interesting"], "cell_type": "markdown"}, {"metadata": {"_uuid": "abf2998339e6238f83ddebdf7430398631c05660", "_cell_guid": "df74e78e-00b0-4162-b45d-4f7fcf6f1695"}, "source": ["### <font color='salmon'> chi-squared Test </font>"], "cell_type": "markdown"}, {"metadata": {"_uuid": "e4963a4bc674bb4e77e798ec4b8b5daea24bc624", "_cell_guid": "6b96f18c-ec0b-4689-89d9-e547ee14a0ab"}, "source": ["#### <font color='navy'> Question 1 - Is there any statistically significant relation between Store Type and Cluster of the stores ?\n", " \n", "* Null Hypothesis H0        = Store Type (a, b, c, d, e) and Cluster (1 to 17) are independent from each other.\n", "* Alternative Hypothesis HA = Store Tpe and cluster are not independent of each other. There is a relationship between them.\n", "\n", " \n", "* Store Type - categorical variable\n", "* Cluster - categorical variable\n", "\n", "Now, to determine if there is a statistically significant correlation between the variables, \n", "we use a chi-square test of independence of variables in a contingency table\n", "\n", "Here, we create a contingency table, with the frequencies of all possible values\n"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "9551fab308915f467f46e3dd32a8212135a777cd", "_cell_guid": "1ca86af4-06d3-410f-a6ad-ecf5b2427547"}, "source": ["# Contingency table\n", "ct = pd.crosstab(store['type'], store['cluster'])\n", "ct"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "3519b442ad6564bbc90da50e49113a3a494bce14", "_cell_guid": "4963b2b6-20bd-4255-af7e-177ad7d0a2d2"}, "source": ["ct.plot.bar(figsize = (15, 6), stacked=True)\n", "plt.legend(title='cluster vs Type')\n", "plt.show()"], "cell_type": "code"}, {"metadata": {"_uuid": "da0c15bc1e9f9ee33e8ffa306b555d10c51de08c", "_cell_guid": "18c08639-1635-4b7d-9a3c-505d32ad6649"}, "source": ["Finally, we compute the chi-square test statistic and the associated p-value. The null hypothesis is the independence between the variables. SciPy implements this test in scipy.stats.chi2_contingency, which returns several objects. We're interested in the second result, which is the p-value."], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "17e052dae5cbac0af682cc493ba6d2a7905d7cb9", "_cell_guid": "b321d2b8-e012-48a2-9556-e811945de81a"}, "source": ["st.chi2_contingency(ct)"], "cell_type": "code"}, {"metadata": {"_uuid": "edd535c7c3fbb65e126cef8768d9771def35dad3", "_cell_guid": "ede93ce3-c1a3-4c80-a63f-8cf1d3052e8e"}, "source": ["####  Interpretation of Result:\n", "\n", "* The p-value is much lower than 0.05.\n", "* There is strong evidence that the null hypothesis is False.\n", "* We reject the null hypothesis and conclude that there is a statistically significant correlation between the Store Type and cluster of the stores."], "cell_type": "markdown"}, {"metadata": {"_uuid": "683b31a0c1c07a1e85cde741a0e10429133345c1", "_cell_guid": "332eae89-6ba1-4321-8e57-f2037868c001"}, "source": ["#### <font color='navy'> Question 1A - Is there any statistically significant relation between City and Cluster of the stores ?\n", " \n", "* Null Hypothesis H0        = City and Cluster are independent from each other.\n", "* Alternative Hypothesis HA = City Tpe and cluster are not independent of each other. There is a relationship between them.\n", "\n", " \n", "* City - categorical variable\n", "* Cluster - categorical variable\n", "\n", "Now, to determine if there is a statistically significant correlation between the variables, \n", "we use a chi-square test of independence of variables in a contingency table\n", "\n", "Here, we create a contingency table, with the frequencies of all possible values"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "44c31edf46f495ade2bf83e7264be8258965942d", "_cell_guid": "217c9a78-8bb1-4e23-aa50-0daf9f3eb5e2"}, "source": ["# Contingency table\n", "ct2 = pd.crosstab(store['city'], store['cluster'])\n", "ct2"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "c069d4b7bc9d509dd2b3b37d7be88798d74b2518", "_cell_guid": "1a45cf5e-6d69-4404-bd98-cdfd723ae5c6"}, "source": ["ct2.plot.bar(figsize = (15, 6), stacked=True)\n", "plt.legend(title='cluster')\n", "plt.show()"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "d3fd5f636fb0979b4bcc2c0732098bf27e554cda", "_cell_guid": "27cfd4f8-ebfa-4308-93e9-1efd33e37193"}, "source": ["st.chi2_contingency(ct2)"], "cell_type": "code"}, {"metadata": {"_uuid": "e3083460161c7f8c72206d52ece1ff2ff1d85823", "_cell_guid": "68bd7c25-4d86-4647-9974-62f6fb4d0006"}, "source": ["####  Interpretation of Result:\n", "\n", "* The p-value is higher than 0.05.\n", "* There is NO evidence to reject Null Hypothesis.\n", "* We can continue with the null hypothesis and conclude that there is no dependence of cluster and cities"], "cell_type": "markdown"}, {"metadata": {"_uuid": "7ffeb4ac27b3e2e43ec72e7dc01983cad72981e7", "_cell_guid": "869c358b-056f-4f51-a3d7-49dc05016e2c"}, "source": ["### <font color='salmon'> t-Test"], "cell_type": "markdown"}, {"metadata": {"_uuid": "15910fd825091e67c7486cc82c4895c4d4020076", "_cell_guid": "e6b09e2b-c836-411a-ad92-c1b278835001"}, "source": ["#### <font color='navy'> Question 2 - Is there any statistically significant relation between Store Type and Sales of the stores ?"], "cell_type": "markdown"}, {"metadata": {"_uuid": "90fce39156250470508435c0c5de7033ef6101fd", "_cell_guid": "e814f07b-359b-40be-b7ad-919a7c8d7802"}, "source": ["* Null Hypothesis H0        = Promotion and Sales are independent from each other.\n", "* Alternative Hypothesis HA = Promotion and Sales are not independent of each other. There is a relationship between them.\n", "\n", "\n", "* Promotion - categorical variable - Independent variable\n", "* Sales - continuous variable - Dependent variable\n", "\n", "Now, to determine if there is a statistically significant correlation between the variables, \n", "we use a student t test\n", "\n", "2-sample t-test: testing for difference across populations"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "746342ff22c83a44129d8b9e05952fd7fa2deec7", "_cell_guid": "cb840976-0025-44d5-9815-9c038dfbec91"}, "source": ["promo_sales = train[train['onpromotion'] == 1.0]['unit_sales']\n", "nopromo_sales = train[train['onpromotion'] == 0.0]['unit_sales']\n", "st.ttest_ind(promo_sales, nopromo_sales, equal_var = False)"], "cell_type": "code"}, {"metadata": {"_uuid": "1724822f21e1e2e82e91120b1c054a2efe32de9b", "_cell_guid": "4b31d1aa-000b-486c-aa40-94472c2e7d1d"}, "source": ["### <font color='salmon'> Correlation / Regression"], "cell_type": "markdown"}, {"metadata": {"_uuid": "8c6ab0a8b44d32aa1968b3de355392669f88df30", "_cell_guid": "e1a91bcd-32fa-440e-864a-24dc0c6ce2d6"}, "source": ["#### <font color='navy'> Question 3 - Is there any statistically significant relation between  Oil price and Sales of the stores ?\n", "\n", "* Null Hypothesis H0        = Oil price and Sales are independent from each other.\n", "* Alternative Hypothesis HA = Oil price and Sales are not independent of each other. There is a relationship between them.\n", "\n", "\n", "* Oil Price - Independent continuous variable\n", "* Sales - Dependent continuous variable\n", "\n", "##### We will do Simple Linear Regression now"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "a11664c51d27ce8c107fd897f381678d6c15db0f", "_cell_guid": "8a985691-7d6e-4e75-8db5-69a2b3f689a2"}, "source": ["lm0 = smf.ols(formula = 'unit_sales ~ dcoilwtico', data = train).fit()"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "0ae0667970500c8c859a626b6814498abcd151df", "_cell_guid": "d29c0b53-feb2-4dd4-a841-26506a087602"}, "source": ["#print the Result \n", "print(lm0.summary())"], "cell_type": "code"}, {"metadata": {"_uuid": "ba8ac2a169a662a12233525455ed93a21f250ab1", "_cell_guid": "784bfb4e-c1b3-46ba-9895-185eccedbc36"}, "source": ["No relation between oil price on sales"], "cell_type": "markdown"}, {"metadata": {"collapsed": true, "_uuid": "5d24ed5d533b3bcf895f5f72ebd2967d86613753", "_cell_guid": "9d5a76ed-84e9-4018-beab-8c979d9439d6"}, "source": ["> ### <font color='salmon'>Random Forest - Trying to predict Sales"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "0fabe73cb0e329990f7b3866fdf776a92ea19626", "_cell_guid": "40e36407-7188-4a56-98ef-642a78cd4775"}, "source": ["X_train = train.drop(['unit_sales', 'description', 'locale_name','locale','city','state','family','type_x','type_y','cluster','class','perishable','transferred', 'dcoilwtico'], axis = 1)\n", "y_train = train.unit_sales"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "e0f98ba00fe573bab6b35579f96c634935e8ee95", "_cell_guid": "441a5547-97a9-46f1-973b-7011b700b037"}, "source": ["rf = RandomForestRegressor(n_jobs = -1, n_estimators = 15)\n", "y = rf.fit(X_train, y_train)\n", "print('model fit')"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "0b2a2c9cf958a89d9a99a83f7f58999cfe64aa04", "_cell_guid": "35473952-cd59-4bbe-866b-be422d7b2024"}, "source": ["X_test = test\n", "y_test = rf.predict(X_test)"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "c35a05aa68412b57cb0da7c67615f0add27cf45d", "_cell_guid": "1e79a9cd-d37c-4e24-9e88-0f2730fa2b3c"}, "source": ["result = pd.DataFrame({'id':test.id, 'unit_sales': y_test}).set_index('id')\n", "result = result.sort_index()\n", "result[result.unit_sales < 0] = 0\n", "result.to_csv('submissionR.csv', index=False)\n", "print('submission created')\n"], "cell_type": "code"}, {"metadata": {"_uuid": "4ca679b8dff1a7a0c47a5de134920133241455d0", "_cell_guid": "0c66ee14-26e6-407f-ada7-de4aa580a974"}, "source": ["### <font color='salmon'> XGBOOST - Try to predict Sales"], "cell_type": "markdown"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "ea93a28e43f25f281f119c1cc5e3edfa92772fd2", "_cell_guid": "e3b8bc97-edb3-4c46-aba5-c632e12380b6"}, "source": ["train_items1 = train_items1.drop(['unit_sales','family','class','perishable'], axis = 1)"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "b67f3a92ed2e44b7fcab02ac2b74efe5825c2755", "_cell_guid": "4d679d31-7cd4-41f8-bc24-b740be785a4c"}, "source": ["train_items1.head()"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "7afd6c7ac5d090458184b237a78e4e3889e55fcc", "_cell_guid": "e4af8e7b-932a-4c44-a02c-db8b58d995d8"}, "source": ["train_items2 = train_items2.drop(['id','store_nbr','item_nbr','onpromotion', 'day_item_purchased','month_item_purchased','quarter_item_purchased','year_item_purchased','family','class','perishable'], axis = 1)"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "345201d83f10e9ac46f67907fa2387460324aada", "_cell_guid": "5634abdb-acc8-4e5b-989b-06479369325a"}, "source": ["train_items2.head()"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "10028197943adf204ff8d52eace530a3b319403d", "_cell_guid": "83dd396e-5eb4-4862-993d-af502d9ebd06"}, "source": ["Xg_train, Xg_valid = train_test_split(train_items1, test_size=0.012, random_state=10)\n", "Yg_train, Yg_valid = train_test_split(train_items2, test_size=0.012, random_state=10)\n", "features = list(train_items1.columns.values)\n", "features2 = list(train_items2.columns.values)"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "da67292dfff2e4504974b4f6bb97b00c8fa127ba", "_cell_guid": "12967425-35cf-411b-864d-69ac7705b7fe"}, "source": ["features "], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "9be85c052d1026014ae865f29a18c8d9f1bd6b0b", "_cell_guid": "083289a9-49f4-4f28-af98-b2cca04c2e85"}, "source": ["features2"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "487f21a8cb97992fe215bd1fb559a56ed63bcb0c", "_cell_guid": "ec19418f-e837-4c67-a05a-c2668905ebc9"}, "source": ["#dtrain = xgb.DMatrix(Xg_train[features], Xg_train.unit_sales)\n", "#dvalid = xgb.DMatrix(Xg_valid[features], Xg_valid.unit_sales)\n", "#Xg_train.dtypes"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "76190678ee76aeda6200f0bff103c01b0e0c618d", "_cell_guid": "33d86cfd-de6d-4782-bd6b-829999dd87aa"}, "source": ["dtrain = xgb.DMatrix(Xg_train[features], Yg_train[features2])\n", "dvalid = xgb.DMatrix(Xg_valid[features], Yg_valid[features2])"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "1647ddf04369bc74df548dfeb9e9c2d7a47ec48c", "_cell_guid": "ac09c10d-0614-4d5a-a2b1-afb84f30d044"}, "source": ["def rmspe(y, yhat):\n", "    return np.sqrt(np.mean((yhat/y-1) ** 2))"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "3f0bda0522978c00a75e8759f6fc6bf6a9e2393f", "_cell_guid": "91d27cff-b849-42dc-a7d3-c155326d497a"}, "source": ["def rmspe_xg(yhat, y):\n", "    y = np.expm1(y.get_label())\n", "    yhat = np.expm1(yhat)\n", "    return \"rmspe\", rmspe(y,yhat)"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "693bc0eeb9eeccf410debc7e5b782e82509dfe25", "_cell_guid": "5e9b0e2e-7847-4e28-a475-6dd292ecf836"}, "source": ["params = {\"objective\": \"reg:linear\",\n", "          \"booster\" : \"gbtree\",\n", "          \"eta\": 0.3,\n", "          \"max_depth\": 10,\n", "          \"subsample\": 0.9,\n", "          \"colsample_bytree\": 0.7,\n", "          \"silent\": 1,\n", "          \"seed\": 1301\n", "          }\n", "num_boost_round = 30\n", "watchlist = [(dtrain, 'train'), (dvalid, 'eval')]\n"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"scrolled": true, "collapsed": true, "_kg_hide-output": true, "_uuid": "4f4ee5d9486b34994f9f0927f03bfea05310b53c", "_cell_guid": "26c74574-d9bc-473a-af97-0de063e6c03c"}, "source": ["gbm = xgb.train(params, dtrain, num_boost_round, evals=watchlist, \\\n", "  early_stopping_rounds=20, feval=rmspe_xg, verbose_eval=True)\n"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "98ab8bc71a80f5728f728f062f431053e48083ca", "_cell_guid": "7450df92-3402-49e9-9776-e87b29035328"}, "source": ["print(\"Validating\")\n", "yhat = gbm.predict(xgb.DMatrix(Xg_valid[features]))\n", "error = rmspe(Yg_valid.unit_sales.values, np.expm1(yhat))\n", "print('RMSPE: {:.6f}'.format(error))"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "ec539e547107e6bef88a117d77657ad463b65acc", "_cell_guid": "d04e4ef5-92d7-42db-a733-1674194faf21"}, "source": ["testg.head()"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "7b92efa88cbe5324da608d38bd099919333ed3d3", "_cell_guid": "f7e5023b-c1e4-46d9-a2f1-9b9c716dcdc1"}, "source": ["testg['date'] = pd.to_datetime(testg['date'], format='%Y-%m-%d')\n", "testg['day_item_purchased'] = testg['date'].dt.day\n", "testg['month_item_purchased'] =testg['date'].dt.month\n", "testg['quarter_item_purchased'] = testg['date'].dt.quarter\n", "testg['year_item_purchased'] = testg['date'].dt.year\n", "testg.drop('date', axis=1, inplace=True)"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "a7a5cdbe3c059f5d9ce0abaee097d724dd3f4e8c", "_cell_guid": "da38f7ac-b04a-4ebb-87f3-31df41ce9de0"}, "source": ["testg.head()"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "9754397a1f606533c8e75e504e15c05640191096", "_cell_guid": "e4a2efb0-38db-406d-8e8a-297533b18f09"}, "source": ["features"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "94355492f25303159efdf4d7b09080000cd75de1", "_cell_guid": "9fdc2fdf-f1f2-4a80-97dd-7505fca8bd03"}, "source": ["testg.loc[(train_items.unit_sales<0),'unit_sales'] = 1 \n", "#testg['unit_sales'] =  train_items['unit_sales'].apply(pd.np.log1p) \n", "testg['onpromotion'] = testg['onpromotion'].astype('category')\n", "cat_columns = testg.select_dtypes(['category']).columns\n", "testg[cat_columns] = testg[cat_columns].apply(lambda x: x.cat.codes)"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "65877edac26fcafe31a42bb22eb6df31c82fc5d3", "_cell_guid": "4412c5c6-1946-4b3f-8f58-d220ba6a4db6"}, "source": ["dtest = xgb.DMatrix(testg[features])"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "e633b428daecc8dacd1fd21f912facee1a73e202", "_cell_guid": "e6cb20e6-da97-41b3-936b-792892b83d68"}, "source": ["test_probs = gbm.predict(dtest)\n", "print(\"Make predictions on the test set\")"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "fad2a7241544b97b214fd691476ed08f8e9a58f6", "_cell_guid": "41f42425-2488-4436-92bf-0d4e1dba0927"}, "source": ["result = pd.DataFrame({\"id\": test[\"id\"], 'unit_sales': np.expm1(test_probs)})\n", "result.to_csv(\"submissionX2.csv\", index=False)\n", "print(\"Submission created\")"], "cell_type": "code"}, {"outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_uuid": "2fe5a46739f498226749fe835f4275d59d9262b7", "_cell_guid": "160c0b23-0442-40ca-8e46-3974179aac31"}, "source": [], "cell_type": "code"}], "metadata": {"language_info": {"mimetype": "text/x-python", "version": "3.6.3", "codemirror_mode": {"version": 3, "name": "ipython"}, "pygments_lexer": "ipython3", "file_extension": ".py", "nbconvert_exporter": "python", "name": "python"}, "kernelspec": {"name": "python3", "display_name": "Python 3", "language": "python"}}, "nbformat": 4, "nbformat_minor": 1}