{"cells": [{"metadata": {"_uuid": "1cac92ee2057a2566eb36a3aab3a7e01c16bb635", "_cell_guid": "c4965afd-92bd-4b34-8765-64490d55649f"}, "source": ["# import relevant libraries\n", "import numpy as np\n", "import pandas as pd\n", "import gc\n", "import random\n", "import xgboost as xgb\n", "from sklearn.cross_validation import train_test_split"], "cell_type": "code", "outputs": [], "execution_count": 1}, {"metadata": {"collapsed": true, "_uuid": "98ab9e1834bca496e84dd8bc5a9e9efedfe2c89f", "_cell_guid": "0d2ec3f0-2d2f-49b2-9118-9e300ad67ef3"}, "source": ["# Select what to skip (random sampling)\n", "n = 125497040 # number of records in file\n", "s = 1000 # sample size\n", "select = sorted(random.sample(range(1, n + 1), s))\n", "skip  = tuple(set(range(1, n + 1)) - set(select))"], "cell_type": "code", "outputs": [], "execution_count": 2}, {"metadata": {"collapsed": true, "_uuid": "f16a910ff96b367e8348b3c9cb00ea1238cae7b3", "_cell_guid": "fbda2243-bccf-4d8e-acfa-3b3d55f1844e"}, "source": ["train = pd.read_csv(\"../input/train.csv\",skiprows = skip)"], "cell_type": "code", "outputs": [], "execution_count": 3}, {"metadata": {"_uuid": "69b91eef17a1759637e36cbe65c41512cdc27db6", "_cell_guid": "4ce755f1-2886-4fae-80cc-f1e773d6ef90"}, "source": ["train.head()"], "cell_type": "code", "outputs": [], "execution_count": 4}, {"metadata": {"_uuid": "2afb8fb81ad6382df56792a81750e51b60749b90", "_cell_guid": "3d4eb227-b404-40b4-a3b6-2c9ac0ed9fcc"}, "source": ["print(train.shape)"], "cell_type": "code", "outputs": [], "execution_count": 5}, {"metadata": {"collapsed": true, "_uuid": "2624846b65d9a67b66ba0e838d599877f82eb427", "_cell_guid": "67121246-b969-4b1f-a677-8807150af85a"}, "source": ["test  = pd.read_csv(\"../input/test.csv\")\n", "store = pd.read_csv(\"../input/stores.csv\")\n", "holiday = pd.read_csv(\"../input/holidays_events.csv\")\n", "item = pd.read_csv(\"../input/items.csv\")\n", "oil = pd.read_csv(\"../input/oil.csv\")\n", "transaction = pd.read_csv(\"../input/transactions.csv\")"], "cell_type": "code", "outputs": [], "execution_count": 6}, {"metadata": {"collapsed": true, "_uuid": "9d18d697c0cca9bd420a976b5cf94402127ee82b", "_cell_guid": "5fb7add6-c40a-4b26-a365-98b60a32e3cd"}, "source": ["merged_train = pd.merge(train, store, on= \"store_nbr\")\n", "merged_train = pd.merge(merged_train, item, on= \"item_nbr\")\n", "merged_train = pd.merge(merged_train, holiday, on=\"date\")\n", "merged_train = pd.merge(merged_train, oil, on =\"date\")"], "cell_type": "code", "outputs": [], "execution_count": 7}, {"metadata": {"_uuid": "5d209662bad08e9d69d94af48e6d98e753e0a424", "_cell_guid": "0498b272-5cb7-47ea-a543-a405da1d80fe"}, "source": ["merged_train.head()"], "cell_type": "code", "outputs": [], "execution_count": 8}, {"metadata": {"collapsed": true, "_uuid": "d5c2c00d5591503cae251fb94c47c2ed64237202", "_cell_guid": "7bb4b71e-0db8-49ce-918c-0cdfeb34f3d8"}, "source": ["train_items = pd.merge(train, item, how='inner')"], "cell_type": "code", "outputs": [], "execution_count": 9}, {"metadata": {"_uuid": "7bf6fdfb4a21a701faf63856263bf24468191494", "_cell_guid": "ba5fcff6-2f6f-4c19-b17c-0c3a809f1878"}, "source": ["train_items.head()"], "cell_type": "code", "outputs": [], "execution_count": 10}, {"metadata": {"_uuid": "cf0f434523af88d2ce5b0b4fd64ea29bd0df9f08", "_cell_guid": "51d19caf-8f85-4b83-ac76-6eceec84ffb8"}, "source": ["oil.head()"], "cell_type": "code", "outputs": [], "execution_count": 11}, {"metadata": {"_uuid": "1d67afbfc21006a6cde86c42f12d4d89f80d8f4a", "_cell_guid": "9520c1a8-dc51-4ea5-8932-0f571fd8c763"}, "source": ["transaction.head()"], "cell_type": "code", "outputs": [], "execution_count": 12}, {"metadata": {"_uuid": "530abb3f533681896fd454785ec5ad924fe1ac75", "_cell_guid": "184c78b0-9916-431a-a1ee-6759fa6e97b1"}, "source": ["item.head()"], "cell_type": "code", "outputs": [], "execution_count": 13}, {"metadata": {"_uuid": "02e28f4aca7435f2aba9e21b93e51615d3e70010", "_cell_guid": "5be99c79-7c38-4a04-a669-d37e1b36451c"}, "source": ["# no. of families (unique) \n", "print(len(item['family'].unique()))"], "cell_type": "code", "outputs": [], "execution_count": 14}, {"metadata": {"_uuid": "0f2cf9c56f3390d8c0c8106970e371fb59357433", "_cell_guid": "1defc02a-07be-4211-acd3-43ce664b7d72"}, "source": ["holiday.head()"], "cell_type": "code", "outputs": [], "execution_count": 15}, {"metadata": {"_uuid": "7e5a276fa68ff1c52b433a3372569279f7ad74f5", "_cell_guid": "40dfe23b-ed48-4087-a19f-fc9479a635a1"}, "source": ["store.head()"], "cell_type": "code", "outputs": [], "execution_count": 16}, {"metadata": {"_uuid": "53fb066a1c9b6c97eb348131e66bcc5d9d5a7111", "_cell_guid": "be65b217-e497-4524-83cf-6ace5309dab1"}, "source": ["# type of stores\n", "print(len(store['type'].unique()))"], "cell_type": "code", "outputs": [], "execution_count": 17}, {"metadata": {"_uuid": "348386133de9a0f20a90b2225e719e4cfdefe0e3", "_cell_guid": "e19d5be2-f375-4c7d-8d48-929aee130b65"}, "source": ["test.head()"], "cell_type": "code", "outputs": [], "execution_count": 18}, {"metadata": {"_uuid": "462fdbc04beffc1d4530cee663d138deb4eaf81a", "_cell_guid": "186d9daf-8fba-4917-a657-bc4f8ce2c0cf"}, "source": ["# Data Preprocessing\n", "oil_nan = (oil.isnull().sum() / oil.shape[0]) * 100\n", "oil_nan"], "cell_type": "code", "outputs": [], "execution_count": 19}, {"metadata": {"_uuid": "94e5c33fd1d2f58a75724d26a26693bd78a59c66", "_cell_guid": "f4dd3781-7967-4e5a-8928-a1f646942ec1"}, "source": ["store_nan = (store.isnull().sum() / store.shape[0]) * 100\n", "store_nan"], "cell_type": "code", "outputs": [], "execution_count": 20}, {"metadata": {"_uuid": "f75f7557aa951145e626a8b3d0bc81f25168cc50", "_cell_guid": "2fb2e9ba-162e-4e84-be72-9d169036ecd6"}, "source": ["item_nan = (item.isnull().sum() / item.shape[0]) * 100\n", "item_nan"], "cell_type": "code", "outputs": [], "execution_count": 21}, {"metadata": {"_uuid": "07ba3f8ae0562ca5dcb61770ab9df390c30fc9b3", "_cell_guid": "b73ba428-cffe-4e92-ba12-040bf74b1da2"}, "source": ["train_nan = (train.isnull().sum() / train.shape[0]) * 100\n", "train_nan"], "cell_type": "code", "outputs": [], "execution_count": 22}, {"metadata": {"collapsed": true, "_uuid": "a25a7d67de705cfb4c1bfc56d90ede5d5e5b51f2", "_cell_guid": "01f12429-abc7-44e4-b852-44a27572d64f"}, "source": ["merged_train['onpromotion'] = merged_train['onpromotion'].fillna(2)\n", "merged_train['onpromotion'] = merged_train['onpromotion'].replace(True,1)\n", "merged_train['onpromotion'] = merged_train['onpromotion'].replace(False,0)"], "cell_type": "code", "outputs": [], "execution_count": 23}, {"metadata": {"_uuid": "6e7d8916c3a934ef3f4dab12b12b1abba992a894", "_cell_guid": "2e412c21-2c50-4bc6-b7a1-f7788060d188"}, "source": ["(merged_train['onpromotion'].unique())"], "cell_type": "code", "outputs": [], "execution_count": 24}, {"metadata": {"collapsed": true, "_uuid": "2f03da5e6b494652294d601f0ff6b387050ac434", "_cell_guid": "f0c9e5f7-5bbb-4c2c-8e97-431662795990"}, "source": ["merged_train['dcoilwtico'] = merged_train['dcoilwtico'].fillna(0)"], "cell_type": "code", "outputs": [], "execution_count": 25}, {"metadata": {"_uuid": "57e4e7e2b8269274b1cd1affc7bb71b2fe5fd7b9", "_cell_guid": "d8fd4ca4-3043-4e0f-a162-bebf381bc2e4"}, "source": ["merged_train['Year']  = merged_train['date'].apply(lambda x: int(str(x)[:4]))\n", "merged_train['Month'] = merged_train['date'].apply(lambda x: int(str(x)[5:7]))\n", "merged_train['date']  = merged_train['date'].apply(lambda x: (str(x)[8:]))\n", "\n", "\n", "test['Year']  = test['date'].apply(lambda x: int(str(x)[:4]))\n", "test['Month'] = test['date'].apply(lambda x: int(str(x)[5:7]))\n", "test['date']  = test['date'].apply(lambda x: (str(x)[8:]))\n", "\n", "train.head()"], "cell_type": "code", "outputs": [], "execution_count": 26}, {"metadata": {"collapsed": true, "_uuid": "a4926f7bbf11cdce3e12f433f122ea2fbcbaefd2", "_cell_guid": "e3c560b2-3e51-4d6e-9253-57a5e0cfba73"}, "source": ["# create 2 copies of train_items\n", "train_items1 = pd.merge(train, item, how='inner')\n", "train_items2 = pd.merge(train, item, how='inner')"], "cell_type": "code", "outputs": [], "execution_count": 27}, {"metadata": {"collapsed": true, "_uuid": "12989b6d5c23d95588fbc606b0be2135822cb783", "_cell_guid": "7fc835ee-bb70-42c6-b9fa-8e2263cbc4ab"}, "source": ["# train_items1\n", "train_items1['date'] = pd.to_datetime(train_items1['date'], format = '%Y-%m-%d')\n", "train_items1['day_item_purchased'] = train_items1['date'].dt.day\n", "train_items1['month_item_purchased'] =train_items1['date'].dt.month\n", "train_items1['quarter_item_purchased'] = train_items1['date'].dt.quarter\n", "train_items1['year_item_purchased'] = train_items1['date'].dt.year\n", "train_items1.drop('date', axis = 1, inplace = True)\n", "\n", "# train_items2\n", "train_items2['date'] = pd.to_datetime(train_items2['date'], format = '%Y-%m-%d')\n", "train_items2['day_item_purchased'] = train_items2['date'].dt.day\n", "train_items2['month_item_purchased'] = train_items2['date'].dt.month\n", "train_items2['quarter_item_purchased'] = train_items2['date'].dt.quarter\n", "train_items2['year_item_purchased'] = train_items2['date'].dt.year\n", "train_items2.drop('date', axis = 1, inplace = True)"], "cell_type": "code", "outputs": [], "execution_count": 28}, {"metadata": {"collapsed": true, "_uuid": "d42d7ddd83c4dcc107a09f8e8d16a4d333fcf952", "_cell_guid": "e022a26a-2719-47e9-9d33-64ca069ead29"}, "source": ["# train_items1\n", "train_items1.loc[(train_items1.unit_sales<0),'unit_sales'] = 1 \n", "train_items1['unit_sales'] =  train_items1['unit_sales'].apply(pd.np.log1p) \n", "train_items1['family'] = train_items1['family'].astype('category')\n", "train_items1['onpromotion'] = train_items1['onpromotion'].astype('category')\n", "train_items1['perishable'] = train_items1['perishable'].astype('category')\n", "category_columns = train_items1.select_dtypes(['category']).columns\n", "train_items1[category_columns] = train_items1[category_columns].apply(lambda x: x.cat.codes)\n", "\n", "# train_items2\n", "train_items2.loc[(train_items2.unit_sales<0),'unit_sales'] = 1 \n", "train_items2['unit_sales'] =  train_items2['unit_sales'].apply(pd.np.log1p) \n", "train_items2['family'] = train_items2['family'].astype('category')\n", "train_items2['onpromotion'] = train_items2['onpromotion'].astype('category')\n", "train_items2['perishable'] = train_items2['perishable'].astype('category')\n", "category_columns = train_items2.select_dtypes(['category']).columns\n", "train_items2[category_columns] = train_items2[category_columns].apply(lambda x: x.cat.codes)"], "cell_type": "code", "outputs": [], "execution_count": 29}, {"metadata": {"_uuid": "cdae453f08525ee4f9292a5b8b12aff50d2d30fe", "_cell_guid": "07223780-5586-4fe4-85b5-7fb29324ccc3"}, "source": ["# train_items1\n", "train_items1 = train_items1.drop(['unit_sales','family','class','perishable'], axis = 1)\n", "train_items1.head()\n", "\n", "# train_items2\n", "train_items2 = train_items2.drop(['id','store_nbr','item_nbr','onpromotion', 'day_item_purchased','month_item_purchased','quarter_item_purchased','year_item_purchased','family','class','perishable'], axis = 1)\n", "train_items2.head()"], "cell_type": "code", "outputs": [], "execution_count": 30}, {"metadata": {"collapsed": true, "_uuid": "c2c871bb7b70c06c44590d149ed5fe6d472a771f", "_cell_guid": "cf0ba209-bd10-440c-9055-7e7d7b32a493"}, "source": ["Xg_train, Xg_valid = train_test_split(train_items1, test_size=0.012, random_state=10)\n", "Yg_train, Yg_valid = train_test_split(train_items2, test_size=0.012, random_state=10)\n", "features1 = list(train_items1.columns.values)\n", "features2 = list(train_items2.columns.values)"], "cell_type": "code", "outputs": [], "execution_count": 31}, {"metadata": {"_uuid": "429a02912dd6edfacf9bc0ba74b4091d33297845", "_cell_guid": "a9effcc3-4d3f-4bcf-b981-07afab758f8e"}, "source": ["features1"], "cell_type": "code", "outputs": [], "execution_count": 32}, {"metadata": {"_uuid": "2367cabbcce76388ad11c3da7f02bc765eb58a43", "_cell_guid": "5ec3f1a6-45d6-42c8-8b3e-68df728381cd"}, "source": ["features2"], "cell_type": "code", "outputs": [], "execution_count": 33}, {"metadata": {"collapsed": true, "_uuid": "b588e42c43e4e2158ead541fab3b34842e143e4c", "_cell_guid": "27f9589f-865e-4938-aca2-7912eed29165"}, "source": ["dtrain = xgb.DMatrix(Xg_train[features1], Yg_train[features2])\n", "dvalid = xgb.DMatrix(Xg_valid[features1], Yg_valid[features2])"], "cell_type": "code", "outputs": [], "execution_count": 34}, {"metadata": {"collapsed": true, "_uuid": "554e200b609da97f4df8bebb682e8e5b293472e3", "_cell_guid": "6e4a7f69-0caa-42a6-932d-53addc897649"}, "source": ["def rmspe(y, yhat):\n", "    return np.sqrt(np.mean((yhat / y-1) ** 2))"], "cell_type": "code", "outputs": [], "execution_count": 35}, {"metadata": {"collapsed": true, "_uuid": "6baafdbaee5da2b8e0a331dbd94c9b2a3f34cf0a", "_cell_guid": "c4507f84-8b57-4553-86a3-033ca1f9419d"}, "source": ["def rmspe_xg(yhat, y):\n", "    y = np.expm1(y.get_label())\n", "    y1 = np.expm1(yhat)\n", "    return \"rmspe\", rmspe(y, yhat)"], "cell_type": "code", "outputs": [], "execution_count": 36}, {"metadata": {"collapsed": true, "_uuid": "532ce3fc52783928df79c8eb6fbe3dd8920c99ab", "_cell_guid": "c3e03fc7-000c-4003-b2d6-aac7d3f0d923"}, "source": ["params = {\"objective\": \"reg:linear\",\n", "          \"booster\" : \"gbtree\",\n", "          \"eta\": 0.3,\n", "          \"max_depth\": 10,\n", "          \"subsample\": 0.9,\n", "          \"colsample_bytree\": 0.7,\n", "          \"silent\": 1,\n", "          \"seed\": 1301\n", "          }\n", "num_boost_round = 15\n", "watchlist = [(dtrain, 'train'), (dvalid, 'eval')]"], "cell_type": "code", "outputs": [], "execution_count": 37}, {"metadata": {"_uuid": "89e4f4e0d91a861efa66af5f70d0a1ecdae24c0a", "_cell_guid": "20112d0e-7288-4fd8-9267-0c59bd6c8587"}, "source": ["gbm = xgb.train(params, dtrain, num_boost_round, evals = watchlist,\n", "  early_stopping_rounds = 5, feval = rmspe_xg, verbose_eval = True)"], "cell_type": "code", "outputs": [], "execution_count": 38}, {"metadata": {"_uuid": "0a68a992cabeffc96b74985bd752636f03d870fb", "_cell_guid": "31aa898c-4e38-4889-a7e9-a1a872268f07"}, "source": ["print(\"Validating\")\n", "yhat = gbm.predict(xgb.DMatrix(Xg_valid[features1]))\n", "error = rmspe(Yg_valid.unit_sales.values, np.expm1(yhat))"], "cell_type": "code", "outputs": [], "execution_count": 39}, {"metadata": {"_uuid": "711b11f6b5438d181acfe43f519874c24e5901f5", "_cell_guid": "164e4e62-50f9-430c-9eb6-a676a0f14893"}, "source": ["test_copy = pd.read_csv(\"../input/test.csv\")\n", "test_copy.head()"], "cell_type": "code", "outputs": [], "execution_count": 40}, {"metadata": {"collapsed": true, "_uuid": "13bbedad1c1423e64ea3c00439d0f4cab36c4a9d", "_cell_guid": "fff07e69-5dae-49a3-a6e1-17a39d105834"}, "source": ["test_copy['date'] = pd.to_datetime(test_copy['date'], format='%Y-%m-%d')\n", "test_copy['day_item_purchased'] = test_copy['date'].dt.day\n", "test_copy['month_item_purchased'] = test_copy['date'].dt.month\n", "test_copy['quarter_item_purchased'] = test_copy['date'].dt.quarter\n", "test_copy['year_item_purchased'] = test_copy['date'].dt.year\n", "test_copy.drop('date', axis=1, inplace=True)"], "cell_type": "code", "outputs": [], "execution_count": 41}, {"metadata": {"_uuid": "5a0b2ed68738027b61afd6c33479824729933d72", "_cell_guid": "25e5aa44-c181-4d31-9305-511604646639"}, "source": ["test_copy.head()"], "cell_type": "code", "outputs": [], "execution_count": 42}, {"metadata": {"_uuid": "891aaf5d1b967525ce5164d404c6db715f3d2880", "_cell_guid": "20e9329d-1528-420f-afad-2da41faf7270"}, "source": ["features1"], "cell_type": "code", "outputs": [], "execution_count": 43}, {"metadata": {"_uuid": "8fbf1a6862d3f56257e0aa3017085e4c1819ee1d", "_cell_guid": "9ab11fb5-e1e2-4666-8926-85d43ff2561e"}, "source": ["train_items.head()"], "cell_type": "code", "outputs": [], "execution_count": 44}, {"metadata": {"collapsed": true, "_uuid": "1a87a8b1f4e466b9eb1aa3c4f254ba054f0d72c4", "_cell_guid": "6551602b-b4af-405c-a955-974cb53c5e37"}, "source": ["test_copy['onpromotion'] = test_copy['onpromotion'].astype('category')\n", "cat_columns = test_copy.select_dtypes(['category']).columns\n", "test_copy[cat_columns] = test_copy[cat_columns].apply(lambda x: x.cat.codes)"], "cell_type": "code", "outputs": [], "execution_count": 45}, {"metadata": {"collapsed": true, "_uuid": "9519834c6509b0f4d605b669e58ca566ea22862d", "_cell_guid": "4f263586-2d69-4921-a123-88c8354dc567"}, "source": ["test_dmatrix = xgb.DMatrix(test_copy[features1])"], "cell_type": "code", "outputs": [], "execution_count": 46}, {"metadata": {"_uuid": "7a219025ecc98d71e186c74f18d8c3bc171a4247", "_cell_guid": "79c7dd39-730f-4154-965f-e8c0f0b3d706"}, "source": ["test_prediction = gbm.predict(test_dmatrix)\n", "print(\"Predictions\")"], "cell_type": "code", "outputs": [], "execution_count": 47}, {"metadata": {"_uuid": "2a1c4ae259bc83eea9c0ccdac1f99188f0adda41", "_cell_guid": "06e222a2-7549-4197-a589-b621c6b7709c"}, "source": ["result = pd.DataFrame({\"id\": test[\"id\"], 'unit_sales': np.expm1(test_prediction)})\n", "result.to_csv(\"submissionXG.csv\", index=False)\n", "print(\"Submission created\")"], "cell_type": "code", "outputs": [], "execution_count": 48}, {"metadata": {"collapsed": true}, "source": [], "cell_type": "code", "outputs": [], "execution_count": null}], "nbformat_minor": 1, "metadata": {"language_info": {"mimetype": "text/x-python", "version": "3.6.3", "codemirror_mode": {"version": 3, "name": "ipython"}, "file_extension": ".py", "name": "python", "pygments_lexer": "ipython3", "nbconvert_exporter": "python"}, "kernelspec": {"name": "python3", "display_name": "Python 3", "language": "python"}}, "nbformat": 4}