{"nbformat": 4, "nbformat_minor": 1, "cells": [{"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output."], "metadata": {"_cell_guid": "8df545ba-2ff5-4b4d-b111-3a95a81ce314", "collapsed": true, "_uuid": "5696eeadee5e6b460e1d187ade8d7a51ae37c139"}}, {"source": ["# **Know your data**\n", "\n", "### Step 1. Import libraries\n", "\n", "### Step 2. Import datasets\n", "\n", "### Step 3. Analysis the data\n", "\n", "## stores, items and transactions in train data\n", "**Time horizon - 2013 - 2017**\n", "\n", "Top 5 unit sales in store 1-9,12, 15, 16, 17, 20,21, 22, 24-31, 33, 36, 37, 38, 40-53 are: \n", "* GROCERY I\n", "* BEVERAGES\n", "* PRODUCE\n", "* DAIRY\n", "* CLEANING\n", "\n", "* Store 11, 14, 23 and 26 also have MEAT as TOP 5 products\n", "* Store 13 has PERSONAL CARE as top 5 products\n", "* Store 32, 35 have EGGS\n", "* Store 18, 34, 39 and 54 have BREAD in their baskets\n", "* Store 10, 19 have DELI as top 5 products.\n", "\n", "Patterns were almost similar also in 2017."], "cell_type": "markdown", "metadata": {"_cell_guid": "acf4cf8f-df67-4b06-9450-650caf78a3e4", "_uuid": "6cd3d6458f9d01c542e845e8bc7ce71ce32d2537"}}, {"source": ["# Missing items in train data that need to be trained separately!!!!\n", "### if we consider all data from 2013-2017, then missing items (#60)  that are needed to be trained \n", "\n", "array([ 313094, 1464607, 1467088, 1467093, 2015646, 2109969, 2112280,\n", "       2112303, 2112461, 2112462, 2112607, 2117783, 2118397, 2120180,\n", "       2120216, 2120506, 2120681, 2120768, 2121319, 2124705, 2125814,\n", "       2126696, 2127024, 2127069, 2127175, 2127206, 2127544, 2127651,\n", "       2127733, 2127825, 2127921, 2127992, 2128628, 2128799, 2129334,\n", "       2129350, 2129387, 2129515, 2129616, 2129678, 2129786, 2129790,\n", "       2129892, 2129994, 2130131, 2130219, 2130265, 2130352, 2130474,\n", "       2130521, 2130526, 2130553, 2131010, 2131572, 2131699, 2132163,\n", "       2132318, 2132945, 2132957, 2134244], dtype=int64)\n", "       \n", "## The number of missing classes in train data is :\n", "- 1005\n", "- 1328\n", "- 1334"], "cell_type": "markdown", "metadata": {"_cell_guid": "e90443aa-805e-406c-8d96-1e022a4f8575", "_uuid": "ac4f913e7b692079a259ea0329a384a66182a094"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["#Load data\n", "# Holiday events metadata\n", "\n", "import pandas as pd\n", "original_data = pd.read_csv('../input/holidays_events.csv')\n", "\n", "original_data.head()\n"], "metadata": {"_kg_hide-input": false, "_cell_guid": "a44daae7-d0df-494d-915d-c0829c2caf09", "collapsed": true, "_uuid": "06d2e2a734844c33d8d693394d39ed377b253ac5"}}, {"source": ["## Load meta data"], "cell_type": "markdown", "metadata": {"_cell_guid": "3aa3f37b-d714-48ec-b7d8-903972d43328", "_uuid": "a48afe69c58a6c0a8c76fbe5ae79fa3eae759979"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# Load data\n", "# Oil meta data\n", "oil_meta_data = pd.read_csv('../input/oil.csv')\n", "print(oil_meta_data.head())\n", "print('shape: ', oil_meta_data.shape)"], "metadata": {"_cell_guid": "f1cfa48c-d170-4c96-ad77-a2eba875bb7d", "collapsed": true, "_uuid": "b08aa97a6adccd920a7d801f7755cb67ed854e95"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["oil_meta_data['dcoilwtico'].plot()"], "metadata": {"_cell_guid": "f0ec67d4-b732-451a-a19e-e40e807542e8", "collapsed": true, "_uuid": "2c559b0500bcfda0bd0f271138342bc76c337f7e"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["%pylab inline\n", "#import matplotlib\n", "#matplotlib.style.use('ggplot')\n", "# plot of oil price over 5 years span\n", "import pandas as pd\n", "pd.set_option('display.mpl_style', 'default') # Make the graphs a bit prettier\n", "figsize(15, 5)\n", "oil_data = pd.read_csv('../input/oil.csv', parse_dates = ['date'], dayfirst=True, index_col='date')\n", "oil_data['dcoilwtico'].plot()\n"], "metadata": {"_cell_guid": "c9f12106-3f67-4e82-977b-e4b0c3ba1d3e", "collapsed": true, "_uuid": "d36ea5f8112b4a8341b40d646a3752da1da88fe1"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# print the oil data\n", "oils = oil_data[['dcoilwtico']]\n", "\n", "oils[:5]\n", "\n"], "metadata": {"_cell_guid": "63ecf332-43fd-4bce-90d8-cd011b678390", "collapsed": true, "_uuid": "55f4e1f2771473110e6d6471e8b905d0e293b14b"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# Indexing the oil data with day and weekday\n", "oils.index # index by date\n", "oils.index.day # index by day\n", "oils.index.weekday # index by weekday"], "metadata": {"_cell_guid": "bb8674dc-60d1-4620-8c42-9460b6262f30", "collapsed": true, "_uuid": "7496ba3f8777297383371d65a67c6df4f0ad46c3"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# add the weekday indexing to the dataframe\n", "oils['weekday'] = oils.index.weekday\n", "oils[:5]"], "metadata": {"_cell_guid": "0712d64a-8ba3-496c-bdee-d6f4416157a3", "collapsed": true, "_uuid": "ea744fd6433ec3faf26e09173f7faa4de2522789"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# whats the oil price throughout the week, not important for this analysis\n", "# Just learning some pandas functionality\n", "weekday_counts = oils.groupby('weekday').aggregate(sum)\n", "print(weekday_counts)\n", "weekday_counts.index = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday']\n", "weekday_counts\n"], "metadata": {"_cell_guid": "221e4b38-dd40-43c0-a636-1fe753899e2a", "collapsed": true, "_uuid": "0ae3da76e0f7d0915e0f4e0a13537a320d628e2f"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["\n", "# Plotting the aggregated weekdays oil price, Saturday, Sunday are excluded\n", "weekday_counts.plot(kind='bar')\n", "\n"], "metadata": {"_cell_guid": "ac426cda-9d17-4043-b6b8-6195f034f946", "collapsed": true, "_uuid": "f87759d5d75bcb7bd0943cb49ee29137b2a345cb"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# Items meta data\n", "import pandas as pd\n", "items_meta_data = pd.read_csv('../input/items.csv')\n", "print(items_meta_data.head())\n", "print('shape: ', items_meta_data.shape)"], "metadata": {"_cell_guid": "59340583-412d-41f8-9e82-2fa2800581e4", "collapsed": true, "_uuid": "842f64286169b4bdd84b1493ea4c2c58177f75d7"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# Number of unique class in item meta data\n", "items_meta_data['class'].unique().shape"], "metadata": {"_cell_guid": "b56e25c9-4fac-4354-bcf7-f13023b41c0b", "collapsed": true, "_uuid": "333d8738b4cf919438791bdaa3980fe6844a2c6a"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["print(items_meta_data['family'].unique().shape) # Number of family = 33\n", "items_family = items_meta_data.groupby('family') # group by family\n", "items_class = items_meta_data.groupby('class') # group by class\n", "items_family_class = items_meta_data.groupby(['family', 'class']) # group by family and class\n", "# print the groupby information\n", "print(items_family.count())\n", "print(items_class.count())\n", "print(items_family_class.count())"], "metadata": {"_cell_guid": "17747c02-2cd5-4675-aaf9-914d696f2ee5", "collapsed": true, "_uuid": "0202730dd51d780499169208301708af308b1972", "scrolled": false}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["%pylab inline\n", "#import matplotlib\n", "#matplotlib.style.use('ggplot')\n", "# which family has the highest number of items\n", "import pandas as pd\n", "pd.set_option('display.mpl_style', 'default') \n", "figsize(15, 5)\n", "items_family['item_nbr'].count().plot(kind = 'bar')\n"], "metadata": {"_cell_guid": "c25d0ad1-eefb-44e8-bf63-c874b227cbc6", "collapsed": true, "_uuid": "463221c95e4c93911ec40dac9ffbaf93b8aba772"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# which class the highest number of items\n", "items_class['item_nbr'].count().plot(kind = 'bar')\n"], "metadata": {"_cell_guid": "f0bb42d2-c90d-4c3e-9857-4382040361fc", "collapsed": true, "_uuid": "613820718a5294fba8ba3ec085823dfeefdc8259", "scrolled": true}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# which group has the highest number of items :)\n", "items_family_class['item_nbr'].count().plot(kind = 'bar')"], "metadata": {"_cell_guid": "5da7a9a6-cab0-411a-b6c0-a572e2bf9dd8", "collapsed": true, "_uuid": "bb79803b1a9d9bf293bdc31e36ba0df080bd7523"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# which are perishable items\n", "items_meta_data.groupby(['perishable', 'family', 'class']).count()"], "metadata": {"_cell_guid": "d1fcbd04-6863-4ee8-8607-a61f7619eb82", "collapsed": true, "_uuid": "3993901557d18e04283105a55f92f71d6367e338"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# which are perishable family\n", "items_perishable_family = items_meta_data.groupby(['perishable', 'family'])\n", "items_perishable_family.count()"], "metadata": {"_cell_guid": "5e4e4055-fc1a-4238-83c3-b2b2d9e09749", "collapsed": true, "_uuid": "96d7ce0a306d4230f0e9f75412b03500c3b5c9b3"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# plot the number of items in perishable family group\n", "items_perishable_family['item_nbr'].count().plot(kind = 'bar')"], "metadata": {"_cell_guid": "5ee0c260-89b5-42ac-b718-ac603c467732", "collapsed": true, "_uuid": "cfc70799b4d4f59a7c80502ff016d6ef715f6586"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# Not so meaningful\n", "#items_meta_data.groupby(['class', 'family']).count()"], "metadata": {"_cell_guid": "68e2e72e-3644-4b05-b4e2-826f0d418011", "collapsed": true, "_uuid": "637e53e5ddc78ffd5580e19d3cf55a1e6fa686ad"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# transaction metda data\n", "transactions_data = pd.read_csv('../input/transactions.csv')\n", "print(transactions_data.head())\n", "transactions_data['date'].unique()\n"], "metadata": {"_cell_guid": "78857019-47bf-48f1-9be0-694e73f1cdd4", "collapsed": true, "_uuid": "ba3295c303242132e7b9d19a67bb83a9760c6a2f"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# Transactions meta data\n", "# Indexing the meta data\n", "# Ploting \n", "# Find the number of unique stores involve in transactions \n", "transactions_meta_data = pd.read_csv('../input/transactions.csv', parse_dates = ['date'], dayfirst=True, index_col='date')\n", "print(transactions_meta_data[1:10])\n", "print('shape: ', transactions_meta_data.shape)\n", "transactions_meta_data.plot()\n", "store_nbr = transactions_meta_data['store_nbr'].unique()\n", "print(store_nbr.shape)\n", "store_nbr # 54\n", "\n"], "metadata": {"_cell_guid": "8d42ec86-9080-454c-aa84-c7c22163cd71", "collapsed": true, "_uuid": "dade24f106070637445034a042c3b47123126555"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# transactions at store 1 and store 2\n", "transactions_group_by_store = transactions_meta_data.groupby(['store_nbr'])\n", "print(transactions_group_by_store.sum())\n", "transactions_at_store_1 = transactions_group_by_store.get_group(1)\n", "transactions_at_store_2 = transactions_group_by_store.get_group(2)\n", "transactions_at_store_44 = transactions_group_by_store.get_group(44)\n", "\n", "fig, axes = plt.subplots(nrows = 3, ncols = 1)\n", "\n", "#start_date = '2013-01-01'\n", "#end_date = '2013-01-31'\n", "#train_data_2013 = train_data.loc[start_date:end_date]\n", "\n", "#train_data_2013['unit_sales'].plot(ax = axes[0])\n", "#axes[0].set_title('2013')\n", "\n", "#start_date = '2014-01-01'\n", "#end_date = '2014-01-31'\n", "#train_data_2014 = train_data.loc[start_date:end_date]\n", "#train_data_2014['unit_sales'].plot(ax = axes[1])\n", "#axes[1].set_title('2014')\n", "#figsize(15, 5)\n", "transactions_at_store_1['transactions'].plot(ax = axes[0])\n", "transactions_at_store_2['transactions'].plot(ax = axes[1])\n", "transactions_at_store_44['transactions'].plot(ax = axes[2])\n"], "metadata": {"_cell_guid": "63017593-04cd-4911-abe7-29f3d1c8a0a6", "collapsed": true, "_uuid": "c739d665493ae86ffbc80199ac77d803b79674a9"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["#transactions_meta_data = pd.read_csv('../input/transactions.csv')\n", "store = transactions_meta_data.groupby(['store_nbr']).get_group(54)\n", "store.groupby('date').sum()\n", "# Train data of store 54 in comments\n", "# Theres an anomaly between the number of transactions in transactions file and the number of unit_sales in the training file???\n", "#           \tid \tstore_nbr \titem_nbr \tunit_sales \tonpromotion\n", "#date \t\t\t\t\t\n", "#2013-01-02 \t606 \t606 \t606 \t606 \t0\n", "#2013-01-03 \t611 \t611 \t611 \t611 \t0\n", "#2013-01-04 \t568 \t568 \t568 \t568 \t0\n", "#2013-01-05 \t612 \t612 \t612 \t612 \t0\n", "#2013-01-06 \t629 \t629 \t629 \t629 \t0"], "metadata": {"_cell_guid": "972b0b58-7c43-40f3-9d00-b55a56e1f20d", "collapsed": true, "_uuid": "ffbffe96f4015f2b509d782023fd95d458acdc39"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# transactions at store 1 and store 2\n", "# Considering all data\n", "transactions_group_by_store = transactions_meta_data.groupby(['store_nbr'])\n", "figsize(15, 70)\n", "fig, axes = plt.subplots(nrows = 54, ncols = 1)\n", "fig.tight_layout()\n", "for i in range(1,55):\n", "    try:\n", "        transactions_at_store = transactions_group_by_store.get_group(i)\n", "        transactions_at_store['transactions'].plot(ax = axes[i - 1])\n", "        axes[i - 1].set_title('Store ' + str(i))\n", "    except KeyError:\n", "        pass\n", "# Interesting phanomenon are observed in store 24, 25, 52 and 53 for example.\n", "print(transactions_at_store.head())"], "metadata": {"_cell_guid": "1a1b5a07-bf7f-43b8-920a-10118838ac13", "collapsed": true, "_uuid": "9fcf6995b9cf9b45d1fea2e44caa256212de18d8"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# transactions at store 1 and store 2\n", "# considering data from 2015\n", "start_date = '2015-01-01'\n", "end_date = '2015-12-31'\n", "transactions_group_by_store = transactions_meta_data.loc[start_date:end_date].groupby(['store_nbr'])\n", "print(transactions_meta_data.head())\n", "\n", "figsize(10, 70)\n", "fig, axes = plt.subplots(nrows = 54, ncols = 1)\n", "fig.tight_layout()\n", "\n", "\n", "for i in range(1,55):\n", "    try:\n", "        transactions_at_store = transactions_group_by_store.get_group(i)\n", "        transactions_at_store['transactions'].plot(ax = axes[i - 1])\n", "        axes[i - 1].set_title('Store ' + str(i))\n", "    except KeyError:\n", "        #print(str(KeyError))\n", "        pass\n", "\n", "#print(transactions_at_store.head())\n", "# Interesting phanomenon are observed in store 24, 25, 52 and 53 for example."], "metadata": {"_cell_guid": "2248cd2d-6a79-4aa0-943a-9405c6f26b46", "collapsed": true, "_uuid": "70160ec2ffc19207ae1d275b7b22412a8977b032"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# at store 12 transactions missing in April, 2015\n", "figsize(15,5)\n", "transactions_at_store_12 = transactions_group_by_store.get_group(12)\n", "transactions_at_store_12['transactions'].plot()\n", "transactions_at_store_12.loc['2015-03-01' : '2015-06-30']"], "metadata": {"_cell_guid": "98b819c3-d3a5-4406-806b-2562cee646ee", "collapsed": true, "_uuid": "8b9b0011105fdbe77f24039af3903762504b5537"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# transactions at store 1 and store 2\n", "# transactions from 2017\n", "start_date = '2017-01-01'\n", "end_date = '2017-12-31'\n", "transactions_group_by_store = transactions_meta_data.loc[start_date:end_date].groupby(['store_nbr'])\n", "transactions_group_by_store.head()\n", "figsize(10, 70)\n", "fig, axes = plt.subplots(nrows = 54, ncols = 1)\n", "fig.tight_layout()\n", "for i in range(1,55):\n", "    transactions_at_store = transactions_group_by_store.get_group(i)\n", "    transactions_at_store['transactions'].plot(ax = axes[i - 1],  subplots = True)\n", "    axes[i - 1].set_title('Store ' + str(i))\n"], "metadata": {"_cell_guid": "936a5ccb-565c-4f8a-94da-2972df93a025", "collapsed": true, "_uuid": "87617c531159148cc90f9e335818fb40f4c330ca"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["figsize(15,5)\n", "# group by by store_nbr\n", "group_by_store_nbr = transactions_meta_data.groupby('store_nbr')\n", "#print(group_by_store_nbr)\n", "group_by_store_nbr.sum().plot(kind='bar')\n", "#print(group_by_store_nbr.sum())"], "metadata": {"_cell_guid": "43ab4d9c-5bf4-4e84-b499-583f6b252079", "collapsed": true, "_uuid": "81b65a232c08f632c7e71d10dd0d8b639b55d5b7"}}, {"source": ["Most transactions occur in Quito, Pichincha (the capital of Ecuador) \n"], "cell_type": "markdown", "metadata": {"_cell_guid": "342ea8ba-9bb1-40a0-bd72-8599699cfb7d", "_uuid": "0609039c4cb92bfa9c37c9f46d754eebba6286d2"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["transactions_meta_data['store_nbr'].unique().shape"], "metadata": {"_cell_guid": "6b32fc94-782a-4533-b038-84f63051b7be", "collapsed": true, "_uuid": "9e089c4cbcacc0e5ec8cf4435b6a75f0b3054f09"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["start_date = '2013-01-01'\n", "end_date   = '2013-12-31'\n", "transactions_2013 = transactions_meta_data.loc[start_date : end_date]\n", "group_by_store_nbr_2013 = transactions_2013.groupby('store_nbr')\n", "\n", "start_date = '2014-01-01'\n", "end_date   = '2014-12-31'\n", "transactions_2014 = transactions_meta_data.loc[start_date : end_date]\n", "group_by_store_nbr_2014 = transactions_2014.groupby('store_nbr')\n", "\n", "start_date = '2015-01-01'\n", "end_date   = '2015-12-31'\n", "transactions_2015 = transactions_meta_data.loc[start_date : end_date]\n", "group_by_store_nbr_2015 = transactions_2015.groupby('store_nbr')\n", "\n", "start_date = '2016-01-01'\n", "end_date   = '2016-12-31'\n", "transactions_2016 = transactions_meta_data.loc[start_date : end_date]\n", "group_by_store_nbr_2016 = transactions_2016.groupby('store_nbr')\n", "\n", "start_date = '2017-01-01'\n", "end_date   = '2017-12-31'\n", "transactions_2017 = transactions_meta_data.loc[start_date : end_date]\n", "group_by_store_nbr_2017 = transactions_2017.groupby('store_nbr')\n", "\n", "# plot\n", "fig, ax = subplots()\n", "group_by_store_nbr_2013.sum().plot(kind = 'bar', color = 'k', ax = ax)\n", "group_by_store_nbr_2014.sum().plot(kind = 'bar', color = 'DarkGreen', ax = ax)\n", "group_by_store_nbr_2015.sum().plot(kind = 'bar', color = 'DarkBlue', ax = ax)\n", "group_by_store_nbr_2016.sum().plot(kind = 'bar', color = 'DarkOrange', ax = ax)\n", "group_by_store_nbr_2017.sum().plot(kind = 'bar', color = 'Gray', ax = ax)\n", "ax.legend (['Year 2013', 'Year 2014', 'Year 2015', 'Year 2016', 'Year 2017'])\n"], "metadata": {"_cell_guid": "0163c834-6fb0-4041-9167-69760cdd7a29", "collapsed": true, "_uuid": "c2871ff3b89175c66c03ada06adf3dbb2d3093ab"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["transactions_at_store_44_year_2013 = group_by_store_nbr_2013.get_group(44)\n", "transactions_at_store_44_year_2013['weekday'] = transactions_at_store_44_year_2013.index.weekday\n", "weekday_counts = transactions_at_store_44_year_2013.groupby('weekday').aggregate(sum)\n", "weekday_counts.index = ['Monday', 'Tuesday', 'WEDNESDAY', 'Thursday', 'Friday', 'Saturday',\n", "                        'Sunday']\n", "weekday_counts.plot(kind = 'bar')"], "metadata": {"_cell_guid": "c679f8ad-5fa0-4498-8f4b-047fac6d494c", "collapsed": true, "_uuid": "7d1b4b96b1e680af28446a211500722125a8882a"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# check the status of the one store for example the whole year\n", "# store_nbr = 1\n", "transactions_at_store_1_year_2013 = group_by_store_nbr_2014.get_group(1)\n", "#print(transactions_at_store_1_2013)\n", "#transactions_at_store_1_2013.plot(kind = 'bar')\n", "transactions_at_store_1_year_2013['weekday'] = transactions_at_store_1_year_2013.index.weekday\n", "weekday_counts = transactions_at_store_1_year_2013.groupby('weekday').aggregate(sum)\n", "print(weekday_counts)\n", "weekday_counts.index = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', \n", "                        'Sunday']\n", "weekday_counts\n", "weekday_counts.plot(kind='bar')"], "metadata": {"_cell_guid": "7ebaf863-be58-4585-bfc1-7dddf9a1ee9f", "collapsed": true, "_uuid": "36f894371b161aab9dbd277fa62e1c301ba3a709"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# transactions and oils price \n", "p1 = oil_data['dcoilwtico'].plot()\n", "transactions_meta_data['transactions'].plot(ax = p1, style = '.')"], "metadata": {"_cell_guid": "ee422063-e6c1-4347-a4cb-42f55ef31c00", "collapsed": true, "_uuid": "fd454501a4a2c8e173c6620c8b4de795810af0b6"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# Load stores meta data\n", "stores_meta_data = pd.read_csv('../input/stores.csv')\n", "print(stores_meta_data.head())\n", "print('shape: ', stores_meta_data.shape)\n", "print(stores_meta_data)\n"], "metadata": {"_cell_guid": "161f7eb1-5cd0-4e43-81c4-cc324cd8095d", "collapsed": true, "_uuid": "957825817c06663ea1c3876cf65a93c219e3d92a"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["cities_states = stores_meta_data.groupby(['state', 'city'])\n", "print(cities_states.count())"], "metadata": {"_cell_guid": "b19879a2-d960-4a26-9e17-38057c1d8a99", "collapsed": true, "_uuid": "70241088128149ead4729f97bb7c4f30db100178"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# Load data\n", "# train data\n", "#train_data = pd.read_csv('../input/train.csv', nrows=100)\n", "#print(train_data.head())\n", "#print('shape: ', train_data.shape)"], "metadata": {"_cell_guid": "f60ce12f-744a-4b7f-b570-816fca14bc4b", "collapsed": true, "_uuid": "8e078e12779f32bf627796e3aca9da4ff398b44a"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["# Shape of the train data\n", "%pylab inline\n", "#import matplotlib\n", "#matplotlib.style.use('ggplot')\n", "\n", "import pandas as pd\n", "pd.set_option('display.mpl_style', 'default') # Make the graphs a bit prettier\n", "\n", "train_data = pd.read_csv('../input/train.csv', nrows = 1000, parse_dates = ['date'], dayfirst = True, index_col = 'date')\n", "print('shape: ', train_data.shape) # (125497040,6)\n", "store_item_groupby = train_data.groupby(['store_nbr', 'item_nbr'])\n", "\n", "#figsize(15, 5)\n", "store_item_groupby['unit_sales'].sum().plot( kind = 'bar')\n"], "metadata": {"_cell_guid": "0bfe9b63-9dea-4498-94f8-57a073d78562", "collapsed": true, "_uuid": "ff1188c09e895bb481cfc71eea679bbd83b8f3cc"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["import pandas as pd\n", "\n", "# read the train data\n", "# parse the data based on year\n", "# delete the variable after saving to csv file\n", "#train_data_reader = pd.read_csv('../input/train.csv', \n", " #                              parse_dates = ['date'], dayfirst = True, index_col = 'date')\n", "\n", "#train_data_reader = pd.read_csv('../input/train.csv',  \n", "#                               parse_dates = ['date'], dayfirst = True, index_col = 'date')\n", "# 2013\n", "#start_date = '2013-01-01'\n", "#end_date = '2013-01-31'\n", "#filename = 'train_2013.csv'\n", "#train_data_2013 = train_data_reader.loc[start_date:end_date]\n", "#train_data_2013.to_csv(filename)\n", "#del train_data_2013\n", "\n"], "metadata": {"_cell_guid": "92416b06-8ea3-4b6c-a427-5c7642938ea6", "collapsed": true, "_uuid": "d038112ab910bc69da1dc8d176fbd8ac36127398"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["\n", "# 2014\n", "#start_date = '2014-01-01'\n", "#end_date = '2014-12-31'\n", "#filename = 'train_2014.csv'\n", "#train_data_2014 = train_data_reader.loc[start_date:end_date]\n", "#train_data_2014.to_csv(filename)\n", "#del train_data_2014\n", "\n", "# 2015\n", "#start_date = '2015-01-01'\n", "#end_date = '2015-12-31'\n", "#filename = 'train_2015.csv'\n", "#train_data_2015 = train_data_reader.loc[start_date:end_date]\n", "#train_data_2015.to_csv(filename)\n", "#del train_data_2015\n", "\n", "\n", "# 2016\n", "#start_date = '2016-01-01'\n", "#end_date = '2016-12-31'\n", "#filename = 'train_2016.csv'\n", "#train_data_2016 = train_data_reader.loc[start_date:end_date]\n", "#train_data_2016.to_csv(filename)\n", "#del train_data_2016\n", "# 2017\n", "#start_date = '2017-01-01'\n", "#end_date = '2017-12-31'\n", "#filename = 'train_2017.csv'\n", "#train_data_2017 = train_data_reader.loc[start_date:end_date]\n", "#train_data_2017.to_csv(filename)\n", "#del train_data_2017\n", "#del train_data_reader\n", "\n", "\n", "\n", "\n", "\n", "\n", "#for train_data in train_data_reader:\n", " #   print(train_data)\n", "\n", "\n", "#print('shape: ', train_data.shape) # (125497040,6)\n", "#train_data_reader.get_chunk(125497040)\n", "\n", "#store_groupby = train_data.groupby([ 'store_nbr'])\n", "\n", "#figsize(15, 5)\n", "#store_groupby['unit_sales'].count().plot()\n"], "metadata": {"_cell_guid": "f52fbdc8-0899-4017-9833-151af92581ae", "collapsed": true, "_uuid": "743a0d29b46c3cc8e96ff70a7d0bad0c4b3293e7"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["#import pandas as pd\n", "#train_1000 = pd.read_csv('train_1000.csv')\n", "#print(train_1000)\n", "# shape of the test data\n", "#test_data = pd.read_csv('../input/test.csv')\n", "#print('shape: ', test_data.shape) # (3370464,5)\n", "#print(test_data.head(5)) "], "metadata": {"_kg_hide-input": true, "_cell_guid": "0507e44c-68e4-416e-b782-e8c81bb009c8", "_kg_hide-output": true, "_uuid": "c6b2391142f484f6e3cc7d9aec7bd68c770d64ce", "collapsed": true}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["import os.path\n", "os.path.exists('train_2013.csv')"], "metadata": {"_cell_guid": "56de4a9c-3111-4946-925b-98bdd7610756", "collapsed": true, "_uuid": "f49afcb36074d80d334197aedf07de8bf0e2ffb2"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["%pylab inline\n", "#import matplotlib\n", "#matplotlib.style.use('ggplot')\n", "\n", "import pandas as pd\n", "pd.set_option('display.mpl_style', 'default') # Make the graphs a bit prettier\n", "figsize(15, 5)\n", "train_data = pd.read_csv('../input/train.csv', parse_dates = ['date'], dayfirst=True, index_col='date')\n", "# 2013\n", "start_date = '2013-01-01'\n", "end_date = '2013-01-31'\n", "train_data_2013 = train_data.loc[start_date:end_date]\n", "train_data_2013['unit_sales'].plot()\n"], "metadata": {"_cell_guid": "9eb7b255-a2ea-4247-9873-400c00e83667", "collapsed": true, "_uuid": "0b5765b9aa992f8bde8674d012afcb78414fe693"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["start_date = '2013-02-01'\n", "end_date = '2013-02-28'\n", "train_data_2013 = train_data.loc[start_date:end_date]\n", "train_data_2013['unit_sales'].plot()"], "metadata": {"_cell_guid": "4fbe33ac-1810-46f8-b009-2cfb255886d0", "collapsed": true, "_uuid": "9665f36b9fedf1b9f7d1b49497e19bfdbd5bf0dc"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["start_date = '2013-03-01'\n", "end_date = '2013-03-31'\n", "train_data_2013 = train_data.loc[start_date:end_date]\n", "train_data_2013['unit_sales'].transpose().plot()"], "metadata": {"_cell_guid": "a95cb1b4-38fb-4dde-a8d3-6c5686d9467c", "collapsed": true, "_uuid": "7b2a2bb198d7a23b322239678347e9878c478852"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["fig, axes = plt.subplots(nrows=1, ncols=2)\n", "\n", "start_date = '2013-01-01'\n", "end_date = '2013-01-31'\n", "train_data_2013 = train_data.loc[start_date:end_date]\n", "\n", "train_data_2013['unit_sales'].plot(ax = axes[0])\n", "axes[0].set_title('2013')\n", "\n", "start_date = '2014-01-01'\n", "end_date = '2014-01-31'\n", "train_data_2014 = train_data.loc[start_date:end_date]\n", "train_data_2014['unit_sales'].plot(ax = axes[1])\n", "axes[1].set_title('2014')"], "metadata": {"_cell_guid": "bdfa163f-bfec-427f-8d20-be21f3f197ba", "collapsed": true, "_uuid": "e25f394aa134152a00be0c84c7c07cc717dec94d"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["start_date = '2017-01-01'\n", "end_date = '2017-08-15'\n", "train_data_2017 = train_data.loc[start_date:end_date]\n", "train_data_2017['unit_sales'].transpose().plot()\n", "train_data_2017.groupby('item_nbr').get_group(96995).shape"], "metadata": {"_cell_guid": "566ec269-ae16-4e17-af81-d38488bfde2c", "collapsed": true, "_uuid": "fab756768979e48bd858c0c0a7255e8a72c073ec"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["import pandas as pd\n", "test_data = pd.read_csv('../input/test.csv', parse_dates = ['date'], dayfirst = True, index_col = 'date')\n", "test_data.head()\n", "test_data.shape\n"], "metadata": {"_cell_guid": "e82134ab-6936-4253-9e1f-71f113eb64b3", "collapsed": true, "_uuid": "14596749eae80d65de7a269f2838cb7c21df0df1"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["test_data['dayofyear'] = test_data.index.dayofyear\n", "test_data.head()"], "metadata": {"_cell_guid": "1e62505e-f29f-4d65-a5f0-afc8a99ce3dd", "collapsed": true, "_uuid": "8082d4b7f9938caf4d4ae5cb4e3ca2d02402374b"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["print(test_data['dayofyear'].unique())\n", "print(test_data.index.unique())\n", "print(test_data['store_nbr'].unique().shape)\n", "print(test_data['item_nbr'].unique())\n", "print(test_data['onpromotion'].unique())"], "metadata": {"_cell_guid": "aae0e524-2493-4432-8d2d-d1c0885260e7", "collapsed": true, "_uuid": "7d9d85fbef0a9b2f5d4605414a8c10a61db10a56"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["%pylab inline\n", "figsize(5,15)\n", "test_data.plot(x = 'date', y = 'item_nbr')"], "metadata": {"_cell_guid": "165ceea0-510b-42b1-8fa0-05ce2142b453", "collapsed": true, "_uuid": "0d69b5671633487b2cec0a78359e699366419262"}}], "metadata": {"kernelspec": {"language": "python", "name": "python3", "display_name": "Python 3"}, "language_info": {"mimetype": "text/x-python", "codemirror_mode": {"version": 3, "name": "ipython"}, "file_extension": ".py", "nbconvert_exporter": "python", "name": "python", "pygments_lexer": "ipython3", "version": "3.6.3"}}}