{"cells": [{"outputs": [], "metadata": {"_uuid": "48cf7476cddd8ccb0def2e455467515f9c6184e1", "_cell_guid": "e1c1449b-3220-47d5-bc9b-72aa135adacc", "collapsed": true}, "cell_type": "code", "source": ["import numpy as np\n", "import pandas as pd\n", "import seaborn as sns\n", "import matplotlib.pyplot as plt\n", "import gc;gc.collect\n", "import datetime\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "fde7fa8a35c5bd538c626a9ab267eacce7251a8e", "_cell_guid": "36e71b25-7963-4850-9652-0f58cd0fe140", "collapsed": true}, "cell_type": "code", "source": ["#holidays_events = pd.read_csv('../input/holidays_events.csv')\n", "#items = pd.read_csv('../input/items.csv')\n", "#oil = pd.read_csv('../input/oil.csv')\n", "sample_submission = pd.read_csv('../input/sample_submission.csv')\n", "#stores = pd.read_csv('../input/stores.csv')\n", "test = pd.read_csv('../input/test.csv')\n", "train = pd.read_csv('../input/train.csv')\n", "#transactions = pd.read_csv('../input/transactions.csv')\n"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "495557c0313b7039cf90ababcd810bdf35075dfd", "_cell_guid": "d59dcb12-8edc-4881-878e-4fa81e45fc6e", "collapsed": true}, "cell_type": "code", "source": ["train[\"date\"] =  pd.to_datetime(train[\"date\"])\n", "train_2016 = train[train[\"date\"].dt.year == 2016]\n", "date_list=np.unique(train_2016[\"date\"])\n", "del train; gc.collect()\n"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "59c2dff800ada10f91b8089d26351b283cdcc5c0", "_cell_guid": "796e20b3-24d2-4acf-a2b3-8c19aedbf4f6", "collapsed": true}, "cell_type": "code", "source": ["#train_2016[\"Year\"]=train_2016[\"date\"].dt.year\n", "#train_2016[\"Month\"]=train_2016[\"date\"].dt.month\n", "#train_2016[\"Day\"]=train_2016[\"date\"].dt.day\n", "#date_list"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "0ac7b306b775ed45bf8e69772a78a44459097630", "_cell_guid": "af70291d-6b5d-486e-9563-d4e0d96fc9ca", "collapsed": true, "scrolled": true}, "cell_type": "code", "source": ["\n", "#train_2016.set_index(\"date\",inplace=True)\n", "train_2016.drop(\"id\",axis=1,inplace=True)\n", "train_2016.drop(\"onpromotion\",axis=1,inplace=True)\n", "train_2016.fillna(0, inplace=True)\n", "print(train_2016.head(5))\n", "\n", "#train_2016.to_csv('train_2016.csv')"], "execution_count": null}, {"source": ["## Memory Reduction "], "cell_type": "markdown", "metadata": {"_uuid": "aa8161138e795c63384ed1fc5b46de239055c173", "_cell_guid": "938287a5-3c91-4893-8429-5eef993d292a"}}, {"outputs": [], "metadata": {"_uuid": "16103f2fe92057e605dbd9a942bea59ef400b3d6", "_cell_guid": "a11b8e0e-c994-4356-9e07-3ebb4917769e", "collapsed": true}, "cell_type": "code", "source": ["memo = train_2016.memory_usage(index=True).sum()\n", "print(memo/ 1024**2,\" MB\")\n"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "116e942f89497923b3ec5e61bed96cfc8d6aa574", "_cell_guid": "c19d02d8-4b2a-4081-81e1-4c08de869cec", "collapsed": true}, "cell_type": "code", "source": ["print(train_2016.dtypes)"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "1d25a298dfa353b237e81a548d38e4c8b9e59e4e", "_cell_guid": "08cdaef4-9bfe-4cb1-b1b3-8f896ba01a61", "collapsed": true}, "cell_type": "code", "source": ["train_2016['store_nbr'] = train_2016['store_nbr'].astype(np.int8)\n", "train_2016['item_nbr'] = train_2016['item_nbr'].astype(np.int32)\n", "train_2016['unit_sales'] = train_2016['unit_sales'].astype(np.int8)\n"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "1405a8b161b59da11748a425446e543c4f0fb4bb", "_cell_guid": "50f51af2-38cd-4f98-a765-93ab02ff0414", "collapsed": true}, "cell_type": "code", "source": ["memo = train_2016.memory_usage(index=True).sum()\n", "print(memo/ 1024**2,\" MB\")\n"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "6e8968419d678a52a28cdaeaa730486b7746236b", "_cell_guid": "7acade28-891e-4192-bb25-b3f039ad7608", "collapsed": true}, "cell_type": "code", "source": ["train_2016.head()"], "execution_count": null}, {"source": ["## Select one item and one store"], "cell_type": "markdown", "metadata": {"_uuid": "37e2e587f400ae1b13c7db1a9a20f6ba1f0caeb8", "_cell_guid": "6b3db7aa-5a71-4729-909d-efb1c5da8a81", "collapsed": true}}, {"outputs": [], "metadata": {"_uuid": "8452856641a6337d1eb86519f0ae83ccf9ce8e51", "_cell_guid": "fd829151-5521-4705-873f-38b868840c27", "collapsed": true}, "cell_type": "code", "source": ["train_2016_25= train_2016[train_2016[\"store_nbr\"] == 25]\n", "train_2016_25_105574= train_2016_25[train_2016_25[\"item_nbr\"] == 105574]\n", "train_2016_25_105574.drop(\"store_nbr\",axis=1,inplace=True)\n", "train_2016_25_105574.drop(\"item_nbr\",axis=1,inplace=True)\n"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "0aa7fbc7e302caf21d20bfc0e780b71153e94618", "_cell_guid": "0cc29835-b3ba-411c-bca7-bf3f88b464d5", "collapsed": true}, "cell_type": "code", "source": ["train_2016_25_105574.plot(figsize=(15, 6))\n", "plt.show()"], "execution_count": null}, {"source": ["We can see there are some data which are not here . "], "cell_type": "markdown", "metadata": {"_uuid": "4d7484281f7943864af773f0a5fd36b10ec4355c", "_cell_guid": "a52df78e-950d-4794-87d0-1d1da04ed9e6"}}, {"outputs": [], "metadata": {"_uuid": "8ba56d9f9f4dc0c03ae61f51e8c2dbfa41939f72", "_cell_guid": "183973e7-1d74-49fc-a8db-567f9b2e41e3", "collapsed": true}, "cell_type": "code", "source": ["df_date=pd.DataFrame(date_list)\n", "df_date = df_date.rename(columns={0: 'date'})\n", "df_date.head()"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "630e85ec15387fe4c56e0dd122d2b70a84389c1b", "_cell_guid": "078d8894-4b07-4e21-9729-9dd2cbf70feb", "collapsed": true}, "cell_type": "code", "source": ["df = pd.merge(df_date, train_2016_25_105574, how='left', on=['date'])\n", "df.fillna(0,inplace=True)\n", "\n", "df.head()"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "6cd2f99c93d887dc69f30d9022f285d5d6d149c7", "_cell_guid": "a0790fd5-1c65-4c70-902e-f6e27f43634b", "collapsed": true}, "cell_type": "code", "source": ["df.plot(figsize=(15, 6))\n", "plt.show()\n", "df.describe()"], "execution_count": null}, {"source": ["## LSTM"], "cell_type": "markdown", "metadata": {"_uuid": "ad8306e1573c36f622b075b058495120e5a1de85", "_cell_guid": "1a7483f2-80c7-42ad-be68-6d040d2edb37"}}, {"outputs": [], "metadata": {"_uuid": "c01a72c8714734e73962871b6291715171bd857c", "_cell_guid": "a72622c4-6c2f-487f-95df-30b448be53de", "collapsed": true}, "cell_type": "code", "source": ["import numpy\n", "import matplotlib.pyplot as plt\n", "import pandas\n", "import math\n", "from keras.models import Sequential\n", "from keras.layers import Dense,Dropout\n", "from keras.layers import LSTM\n", "from sklearn.preprocessing import MinMaxScaler\n", "from sklearn.metrics import mean_squared_error"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "add7a144a801c5f27d0047aa4849ef6f9b246277", "_cell_guid": "b6dd53b2-498b-4f42-91b3-4aed9e815b95", "collapsed": true}, "cell_type": "code", "source": ["# fix random seed for reproducibility\n", "numpy.random.seed(7)\n", "\n", "\n"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "b63805c5cce11b273c20a4368593130d290709dd", "_cell_guid": "eb4919a3-dcfd-4f73-b895-07d2054610de", "collapsed": true}, "cell_type": "code", "source": ["# split into train and test sets\n", "train, test =df[df[\"date\"].dt.month <= 11], df[df[\"date\"].dt.month > 11]\n", "train=train.set_index(\"date\")\n", "test=test.set_index(\"date\")\n", "df=df.set_index(\"date\")\n", "train=train.values\n", "test=test.values\n", "df=df.values\n"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "73370b1d66a68f15367adb4b3144aca60b832406", "_cell_guid": "5627fa53-c397-42fd-9e42-b297bc1db94a", "collapsed": true}, "cell_type": "code", "source": ["def create_dataset(dataset, look_back=1):\n", "    dataX, dataY = [], []\n", "    for i in range(len(dataset)-look_back-1):\n", "        a = dataset[i:(i+look_back), 0]\n", "        dataX.append(a)\n", "        dataY.append(dataset[i + look_back, 0])\n", "    return numpy.array(dataX), numpy.array(dataY)"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "9f444f568b0396e7eb780906bbec50365c1b95f4", "_cell_guid": "9cccc38b-e7db-48f6-acb8-52f735f452a5", "collapsed": true}, "cell_type": "code", "source": ["# reshape into X=t and Y=t+1\n", "look_back = 4\n", "trainX, trainY = create_dataset(train, look_back)\n", "testX, testY = create_dataset(test, look_back)"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "f2b3be5feccd4180778c37a2222d35c0a970fb66", "_cell_guid": "09349da9-3235-4a77-9d9e-b0465073e56e", "collapsed": true}, "cell_type": "code", "source": ["\n", "# reshape input to be [samples, time steps, features]\n", "trainX = numpy.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1]))\n", "testX = numpy.reshape(testX, (testX.shape[0], 1, testX.shape[1]))"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "8e7b023aa8048a767f339bd0fe869d771a1dd346", "_cell_guid": "a9fdc402-e1b1-4a1f-a1c0-85cee260ed10", "collapsed": true}, "cell_type": "code", "source": ["# create and fit the LSTM network\n", "model = Sequential()\n", "model.add(LSTM(4, input_shape=(1, look_back)))\n", "model.add(Dense(10))\n", "model.add(Dropout(0.2))\n", "model.add(Dense(1))\n", "model.compile(loss='mean_squared_error', optimizer='adam')\n", "model.fit(trainX, trainY, validation_data=(testX, testY),epochs=25, batch_size=1, verbose=2)"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "6c62b817e07bf69081c0d56e75f0ccb12523dd20", "_cell_guid": "ecc2c1c7-94ba-4734-b8db-ea73d53d26d8", "collapsed": true}, "cell_type": "code", "source": ["# make predictions\n", "trainPredict = model.predict(trainX)\n", "testPredict = model.predict(testX)"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "4028ed36bb1ba0019e1d37315f7f2d2c64ab0b55", "_cell_guid": "2baaede9-ef7d-48fa-b4d4-90cc41962558", "collapsed": true}, "cell_type": "code", "source": ["# shift train predictions for plotting\n", "trainPredictPlot = numpy.empty_like(df)\n", "trainPredictPlot[:, :] = numpy.nan\n", "trainPredictPlot[look_back:len(trainPredict)+look_back, :] = trainPredict\n", "# shift test predictions for plotting\n", "testPredictPlot = numpy.empty_like(df)\n", "testPredictPlot[:, :] = numpy.nan\n", "testPredictPlot[len(trainPredict)+(look_back*2)+1:len(df)-1, :] = testPredict\n", "# plot baseline and predictions\n", "\n", "plt.plot(df)\n", "plt.plot(trainPredictPlot)\n", "plt.plot(testPredictPlot)\n", "plt.show()"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "d8a1e2a8eed220996dfbc9ed648c8b5259ceaa42", "_cell_guid": "7d584319-5e46-439c-adf4-8f9451e9a4a3", "collapsed": true}, "cell_type": "code", "source": ["result=pd.DataFrame(testPredict)\n", "result[\"testY\"]=testY\n", "result"], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "c659c292e8709d44c03e71cb550d6ddf068f92d7", "_cell_guid": "b8382a1a-2162-41b6-991e-20eca1ff6c53", "collapsed": true}, "cell_type": "code", "source": [], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "1d8b65b711f7daddc40390533951cab45a463158", "_cell_guid": "6238aeb1-347d-48a8-baa6-9fb672118cf2", "collapsed": true}, "cell_type": "code", "source": [], "execution_count": null}, {"outputs": [], "metadata": {"_uuid": "4cab996b99c9cfba5b282914eee5501bd22f569c", "_cell_guid": "ba72dc19-c877-4ca1-b238-8378d5d06048", "collapsed": true}, "cell_type": "code", "source": [], "execution_count": null}], "nbformat_minor": 1, "nbformat": 4, "metadata": {"kernelspec": {"language": "python", "display_name": "Python 3", "name": "python3"}, "language_info": {"codemirror_mode": {"version": 3, "name": "ipython"}, "nbconvert_exporter": "python", "version": "3.6.3", "file_extension": ".py", "pygments_lexer": "ipython3", "name": "python", "mimetype": "text/x-python"}}}