{"nbformat_minor":1,"cells":[{"cell_type":"markdown","source":"**Importing all the libraries that we will need**","metadata":{"_cell_guid":"8ec0a466-54e6-40cf-b10c-f2edcc069aea","_uuid":"677b4e44df05de3c4fc72be4f8a71c2c7b2045b0"}},{"cell_type":"code","outputs":[],"source":"#Import necessary packages for data analysis\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn import preprocessing, linear_model, metrics\nimport gc; gc.enable()\nimport seaborn as sns\nsns.set(style = 'whitegrid', color_codes = True)\n%matplotlib inline\n#Machine Learning Algorithms\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.linear_model import LinearRegression\n\nfrom operator import add # elementwise addition\nimport folium # plotting data on interactive maps","metadata":{"_cell_guid":"97c999ce-4bdc-4e60-a1e4-9c5352d91349","_uuid":"02fea094594795baa6401ad79899a281e56d5374","collapsed":true},"execution_count":30},{"cell_type":"markdown","source":"**Loading the Data**","metadata":{"_cell_guid":"c4c8fc70-22b3-4c48-b5c3-754424736a53","_uuid":"33ac9c186d24ba835eb936d7870d20c46196c5e8"}},{"cell_type":"code","outputs":[],"source":"PATH = (\"../input/favorita-grocery-sales-forecasting\")\nfrom subprocess import check_output\nprint(check_output([\"ls\", PATH]).decode(\"utf8\"))","metadata":{"_cell_guid":"dbf39ac0-2aef-4299-bb5e-69919eea3597","_uuid":"5a04e480f8b76a3caf1195f9200b05079367ea32"},"execution_count":2},{"cell_type":"code","outputs":[],"source":"dtypes = {'id':'int64', 'item_nbr':'int32', 'store_nbr':'int8', 'onpromotion':str}\ndata = {\n    'train_1': pd.read_csv('../input/favorita-grocery-sales-forecasting/train.csv', dtype=dtypes, parse_dates=['date']),\n    'test': pd.read_csv('../input/favorita-grocery-sales-forecasting/test.csv', dtype=dtypes, parse_dates=['date']),\n    'items': pd.read_csv('../input/favorita-grocery-sales-forecasting/items.csv'),\n    'stores': pd.read_csv('../input/favorita-grocery-sales-forecasting/stores.csv'),\n    'trans': pd.read_csv('../input/favorita-grocery-sales-forecasting/transactions.csv', parse_dates=['date']),\n    'holidays': pd.read_csv('../input/favorita-grocery-sales-forecasting/holidays_events.csv', dtype={'transferred':str}, parse_dates=['date']),\n    'oil': pd.read_csv('../input/favorita-grocery-sales-forecasting/oil.csv', parse_dates=['date']),\n    }","metadata":{"_cell_guid":"18a24ba5-5f84-45d1-9445-9ee098cb7c18","_uuid":"7d7cc8ba8d4256ce94fe5cf9de198edcedf7fd24","collapsed":true},"execution_count":3},{"cell_type":"markdown","source":"**Data processing**","metadata":{"_cell_guid":"6b09b771-18c1-43c6-84a1-07d96c34708e","_uuid":"103a226a1ea39de7078343c63fa7e9c4bbcf0323"}},{"cell_type":"code","outputs":[],"source":"train = data['train_1'][(data['train_1']['date'].dt.month == 8) & (data['train_1']['date'].dt.day > 15)]\ndel data['train_1']; gc.collect();\ntarget = train['unit_sales'].values\ntarget[target < 0.] = 0.\ntrain['unit_sales'] = np.log1p(target)\n\ndef df_lbl_enc(df):\n    for c in df.columns:\n        if df[c].dtype == 'object':\n            lbl = preprocessing.LabelEncoder()\n            df[c] = lbl.fit_transform(df[c])\n            print(c)\n    return df\n\ndef df_transform(df):\n    df['date'] = pd.to_datetime(df['date'])\n    df['yea'] = df['date'].dt.year\n    df['mon'] = df['date'].dt.month\n    df['day'] = df['date'].dt.day\n    df['date'] = df['date'].dt.dayofweek\n    df['onpromotion'] = df['onpromotion'].map({'False': 0, 'True': 1})\n    df['perishable'] = df['perishable'].map({0:1.0, 1:1.25})\n    df = df.fillna(-1)\n    return df\n#Items data\ndata['items'] = df_lbl_enc(data['items'])\ntrain = pd.merge(train, data['items'], how='left', on=['item_nbr'])\ntest = pd.merge(data['test'], data['items'], how='left', on=['item_nbr'])\ndel data['test']; gc.collect();\ndel data['items']; gc.collect();\n#Transactions data\ntrain = pd.merge(train, data['trans'], how='left', on=['date','store_nbr'])\ntest = pd.merge(test, data['trans'], how='left', on=['date','store_nbr'])\ndel data['trans']; gc.collect();\ntarget = train['transactions'].values\ntarget[target < 0.] = 0.00015\ntrain['transactions'] = np.log1p(target)\n#Stores data\ndata['stores'] = df_lbl_enc(data['stores'])\ntrain = pd.merge(train, data['stores'], how='left', on=['store_nbr'])\ntest = pd.merge(test, data['stores'], how='left', on=['store_nbr'])\ndel data['stores']; gc.collect();\n#Holidays data\ndata['holidays'] = data['holidays'][data['holidays']['locale'] == 'National'][['date','transferred']]\ndata['holidays']['transferred'] = data['holidays']['transferred'].map({'False': 0, 'True': 1})\ntrain = pd.merge(train, data['holidays'], how='left', on=['date'])\ntest = pd.merge(test, data['holidays'], how='left', on=['date'])\ndel data['holidays']; gc.collect();\n#Oil data\ntrain = pd.merge(train, data['oil'], how='left', on=['date'])\ntest = pd.merge(test, data['oil'], how='left', on=['date'])\ndel data['oil']; gc.collect();\n#Join data\ntrain = df_transform(train)\ntest = df_transform(test)\ncol = [c for c in train if c not in ['id', 'unit_sales','perishable','transactions']]\nx1 = train[(train['yea'] != 2016)]\nx2 = train[(train['yea'] == 2016)]\ndel train; gc.collect();\ny1 = x1['transactions'].values\ny2 = x2['transactions'].values","metadata":{"_cell_guid":"eb5bf6e6-0d4f-4e6b-b842-fa5493f8c097","_uuid":"565c8d2ce84d240b7e076bc3764ec56d96ae7cde"},"execution_count":4},{"cell_type":"code","outputs":[],"source":"x1.head()","metadata":{"_cell_guid":"bc9e70e8-0014-479c-9fe3-f94da6c05f52","_uuid":"14db6906fbd21bd795cead91c1153dd29ebbef31"},"execution_count":5},{"cell_type":"code","outputs":[],"source":"x2.head()","metadata":{"_cell_guid":"a55b500f-476b-4673-8927-45edfb805b38","_uuid":"7aa01b9811f8523956a4c9247999f8399b5f318f"},"execution_count":6},{"cell_type":"markdown","source":"*Unit sales per promotion*","metadata":{"_cell_guid":"dc8887ed-05cc-4b8b-80c5-000646894372","_uuid":"955fa065864488d65dcad0e8cc48abea76566e50"}},{"cell_type":"code","outputs":[],"source":"fig, (axis1) = plt.subplots(1,1,figsize=(30,4))\nsns.barplot(x='onpromotion', y='unit_sales', data=x1, ax=axis1)","metadata":{"_cell_guid":"d3d4198d-6b2c-43c1-ae26-b5b911753dc0","_uuid":"9b29c58992c8200196d30371b7dd2455c6ee4f1b"},"execution_count":7},{"cell_type":"markdown","source":"*Unit sales per store type*","metadata":{"_cell_guid":"e4c6f1e3-b6b0-4d41-bee9-3c00d083b80f","_uuid":"a3ca44a0f4774fb1c418936bc8b6ddb9818aaff6"}},{"cell_type":"code","outputs":[],"source":"fig, (axis1) = plt.subplots(1,1,figsize=(30,4))\nsns.barplot(x='type', y='unit_sales', data=x1, ax=axis1)","metadata":{"_cell_guid":"835548a3-7721-46a7-ba51-94f7622c84c5","_uuid":"19dac7930b6b1634b75be0acc3930bc0143830ec"},"execution_count":8},{"cell_type":"markdown","source":"*Unit sales per *","metadata":{"_cell_guid":"3762f048-ad80-4f1b-be4d-66026b290c99","_uuid":"97325cca7e0705b00832eb170b641a1ad2038fe2"}},{"cell_type":"code","outputs":[],"source":"fig, (axis1) = plt.subplots(1,1,figsize=(30,4))\nsns.countplot(x=x2['city'], data=x2, ax=axis1)","metadata":{"_cell_guid":"4e75a580-50e4-4dc2-9754-3c00babaf5ef","_uuid":"ad99c46bb3be879ac6af2d7efb1102a58225ccb3"},"execution_count":9},{"cell_type":"markdown","source":"*Transactions*","metadata":{"_cell_guid":"86d2a33d-b2f1-498c-ad3b-0b16bf06db2a","_uuid":"e4a78017e36aaa350295502122f674d956e853f6"}},{"cell_type":"code","outputs":[],"source":"t=x2['transactions']\nfig, (axis1) = plt.subplots(1,1,sharex=True,figsize=(15,8))\nax1 = t.plot(legend=True,ax=axis1,marker='o',title=\"Transactions\")","metadata":{"_cell_guid":"23e6d59b-57e9-44ee-9426-2b242e2041f0","_uuid":"bf0624b743606185a237395bf38271e3491da87b"},"execution_count":10},{"cell_type":"markdown","source":"*Oil price *","metadata":{"_cell_guid":"7997a640-0726-4cbb-9b3e-caf167e009ec","_uuid":"c0d0184101e57eebc48ca608b00f177b9b3d3560"}},{"cell_type":"code","outputs":[],"source":"oil_plot = x2['dcoilwtico'].copy()\noil_plot.index = x2['day'].copy()\noil_plot.plot(figsize=(30,10))\nplt.show()","metadata":{"_cell_guid":"fdb90ba0-858b-4524-8c0e-39962a1dac56","_uuid":"5d3f41a172c2b1a6348067cb9998c000c996eb23","scrolled":true},"execution_count":11},{"cell_type":"markdown","source":"**Predict Sales : Random Forest**","metadata":{"_cell_guid":"ca1d7127-b627-40ec-9cc9-026a78ec8b31","_uuid":"55356aa00c545fba7a8bd3c8fa38e1542b4f6886"}},{"cell_type":"code","outputs":[],"source":"r3 = RandomForestRegressor(n_estimators=86, max_depth=4, n_jobs=-1, \n                                    verbose=0, warm_start=True)\n\nr4 = RandomForestRegressor(n_estimators=78, max_depth=3, n_jobs=-1, \n                                    verbose=0, warm_start=True)","metadata":{"_cell_guid":"55de44cf-9a89-423a-8f2f-55aa75d55a18","_uuid":"f23364236d8e51ea3a466ca9bb531660afbe1491","collapsed":true},"execution_count":13},{"cell_type":"code","outputs":[],"source":"#Initialization\ndef NWRMSLE(y, pred, w):\n    return metrics.mean_squared_error(y, pred, sample_weight=w)**0.5\n#Model 1\nr3.fit(x1[col], y1)\na3 = NWRMSLE(y2, r3.predict(x2[col]), x2['perishable'])\n\n#Model 2\nr4.fit(x1[col], y1)\na4 = NWRMSLE(y2, r4.predict(x2[col]), x2['perishable'])\n\nN3 = str(a3)\nN4 = str(a4)\n\ntest['transactions'] = r3.predict(test[col])\ntest['transactions'] = r4.predict(test[col])\ncol = [c for c in x1 if c not in ['id', 'unit_sales','city','cluster','perishable']]\ny1 = x1['unit_sales'].values\ny2 = x2['unit_sales'].values\n\nr3 = RandomForestRegressor(n_estimators=79, max_depth=5, n_jobs=-1, \n                                    verbose=0, warm_start=True)\n\nr4 = GradientBoostingRegressor(n_estimators=120, max_depth=3, learning_rate = 0.05, \n                                       verbose=0, warm_start=True,\n                                       subsample= 0.65, max_features = 0.35)\n#Fit a random forest classifier to our training set\nr3.fit(x1[col], y1)\nr4.fit(x1[col], y1)\na3 = NWRMSLE(y2, r3.predict(x2[col]), x2['perishable'])\na4 = NWRMSLE(y2, r4.predict(x2[col]), x2['perishable'])\nprint('model fit')","metadata":{"_cell_guid":"9c901f4d-70de-4fb7-b136-648716c4a31c","_uuid":"86f4fbff1e8119e10adaf87a24aec9f8f3e3a773"},"execution_count":14},{"cell_type":"markdown","source":"*Save Predictions Model 1*","metadata":{"_cell_guid":"e4de5c5d-53a3-418a-b28c-57249bb25372","_uuid":"91dda6dd97dc19628b770822af5f99c2e0daf5c4"}},{"cell_type":"code","outputs":[],"source":"#Output file\nN3 = str(a3)\nprint('Accuracy = ',a3*100)\ntest['unit_sales'] = r3.predict(test[col])\ncut = 0.+1e-12 # 0.+1e-15\ntest['unit_sales'] = (np.exp(test['unit_sales']) - 1).clip(lower=cut)\n\noutput_file = 'sample_submission_RF1.csv'\n \ntest[['id','unit_sales','city','cluster']].to_csv(output_file, index=False, float_format='%.2f')\nprint('file created')","metadata":{"_cell_guid":"79b6f5f0-d83a-499b-8db1-0e6398529c2f","_uuid":"7722da66e4d229883a258ecac2fed5941b8d6b69"},"execution_count":15},{"cell_type":"markdown","source":"*Save Predictions Model 2*","metadata":{"_cell_guid":"cfc9feec-b3bb-43d0-9b4c-632b079ef110","_uuid":"4ba5047311d6737345a74df8859573ccae616676"}},{"cell_type":"code","outputs":[],"source":"# part of the output file name\nN4 = str(a4)\nprint('Accuracy = ',a4*100)\ntest['unit_sales'] = r4.predict(test[col])\ncut = 0.+1e-12 # 0.+1e-15\ntest['unit_sales'] = (np.exp(test['unit_sales']) - 1).clip(lower=cut)\n\noutput_file = 'sample_submission_FR2.csv'\n \ntest[['id','unit_sales','city','cluster']].to_csv(output_file, index=False, float_format='%.2f')\nprint('file created')","metadata":{"_cell_guid":"0e2ac57c-f568-4171-9a3f-ee4fe2db6052","_uuid":"585d01869ea96565aceb268fcc6552d5c7f79b34"},"execution_count":16},{"cell_type":"markdown","source":"**Predict Sales : GradientBoosting**","metadata":{"_cell_guid":"5ec72b50-3904-416d-812d-3f85990c5f81","_uuid":"1c2e05a80fe61fc20c30edba767e1405869eacce"}},{"cell_type":"code","outputs":[],"source":"r5 = GradientBoostingRegressor(n_estimators=110, max_depth=3, learning_rate = 0.05, \n                                       verbose=0, warm_start=True,\n                                       subsample= 0.7, max_features = 0.35)\n\nr6 = GradientBoostingRegressor(n_estimators=125, max_depth=4, learning_rate = 0.05, \n                                        verbose=0, warm_start=True,\n                                       subsample= 0.85, max_features = 0.40) ","metadata":{"_cell_guid":"6f37c4bd-cd8c-46a2-9cf0-82bfac4b2fac","_uuid":"21497da4b265def868286803dc835e1d4ce36a7c","collapsed":true},"execution_count":17},{"cell_type":"code","outputs":[],"source":"r5.fit(x1[col], y1)\nr6.fit(x1[col], y1)\na5 = NWRMSLE(y2, r5.predict(x2[col]), x2['perishable'])\na6 = NWRMSLE(y2, r6.predict(x2[col]), x2['perishable'])\nN5 = str(a5)\nN6 = str(a6)\ntest['transactions'] = r5.predict(test[col])\ntest['transactions'] = r6.predict(test[col])\ntest['transactions'] = test['transactions'].clip(lower=0.+1e-15)\n\ncol = [c for c in x1 if c not in ['id', 'unit_sales','city','cluster','perishable']]\ny1 = x1['unit_sales'].values\ny2 = x2['unit_sales'].values\nprint('model fit')","metadata":{"_cell_guid":"b60bc1b3-bb69-4eda-8f3d-73ca27137d0b","_uuid":"730c7951c31f03be3053459a831663e63b4b427a"},"execution_count":18},{"cell_type":"markdown","source":"*Save Predictions Model 1*","metadata":{"_cell_guid":"6beb7fcf-e7f0-41dd-ad51-fcd4c84dd7ec","_uuid":"867eb727423cb5c28326d246fdc5157c4ae0dbc6"}},{"cell_type":"code","outputs":[],"source":"N5 = str(a5)\nprint('Precisions = ',a5)\ntest['unit_sales'] = r5.predict(test[col])\ncut = 0.+1e-12 # 0.+1e-15\ntest['unit_sales'] = (np.exp(test['unit_sales']) - 1).clip(lower=cut)\n\noutput_file = 'sample_submission_GB1.csv'\n \ntest[['id','unit_sales']].to_csv(output_file, index=False, float_format='%.2f')\nprint('file created')","metadata":{"_cell_guid":"d3a85ea2-982d-4ef0-904e-d3180911dd35","_uuid":"a663bfd3cda97d873883a7652efad6d3bb096cf6"},"execution_count":19},{"cell_type":"markdown","source":"*Save Predictions Model 2*","metadata":{"_cell_guid":"a04f6a39-ee71-47ea-8361-4b0ed23d0a1e","_uuid":"5d720a39d3ec0f660410c7f29a440420be005518"}},{"cell_type":"code","outputs":[],"source":"N6 = str(a6)\nprint('Precisions = ',a6)\ntest['unit_sales'] = r6.predict(test[col])\ncut = 0.+1e-12 # 0.+1e-15\ntest['unit_sales'] = (np.exp(test['unit_sales']) - 1).clip(lower=cut)\n\noutput_file = 'sample_submission_GB2.csv'\n \ntest[['id','unit_sales','cluster','city','cluster']].to_csv(output_file, index=False, float_format='%.2f')\nprint('file created')","metadata":{"_cell_guid":"49610453-70f3-43eb-971c-d21843b4b1cb","_uuid":"03177930d7b995a2774f88cfa92703821abfd85b"},"execution_count":20},{"cell_type":"code","outputs":[],"source":"pred_by_city1 = pd.read_csv('sample_submission_RF1.csv')\npred_by_city1.head()","metadata":{"_cell_guid":"53a887d1-02c2-4cfb-9e88-d4cab27c006e","_uuid":"4341fbc6307e25472d4790d2e15359a1444a6387"},"execution_count":28},{"cell_type":"code","outputs":[],"source":"pred_by_city2 = pd.read_csv('sample_submission_FR2.csv')\npred_by_city2.head()","metadata":{},"execution_count":27},{"cell_type":"code","outputs":[],"source":"pred_by_city3 = pd.read_csv('sample_submission_GB2.csv')\npred_by_city3.head()","metadata":{"_cell_guid":"f15f2da1-0c63-4c43-8028-db6480ac35d4","_uuid":"4b37ba76b11aa3fe8b5afacbe4c1848add438588"},"execution_count":24},{"cell_type":"code","outputs":[],"source":"store_locations={\n 'Ambato' : [-1.2543408,-78.6228504],\n 'Babahoyo' : [-1.801926,-79.53464589999999],\n 'Cayambe' : [0.025,-77.98916659999998],\n 'Cuenca' : [-2.9001285,-79.0058965],\n 'Daule' : [-1.86218,-79.97766899999999],\n 'El Carmen' : [-0.266667, -79.4333],\n 'Esmeraldas' : [0.9681788999999998,-79.6517202],\n 'Guaranda' : [-1.5904721,-78.9995154],\n 'Guayaquil' : [-2.1709979,-79.92235920000002],\n 'Ibarra' : [0.3391763,-78.12223360000002],\n 'Latacunga' : [-0.7754954,-78.52064999999999],\n 'Libertad' : [-2.2344458,-79.91122430000001],\n 'Loja' : [-4.0078909,-79.21127690000003],\n 'Machala' : [-3.2581112,-79.9553924],\n 'Manta' : [-0.9676533,-80.70891010000003],\n 'Playas' : [-2.6284683,-80.38958860000002],\n 'Puyo' : [-1.4923925,-78.00241340000002],\n 'Quevedo' : [-1.0225124,-79.46040349999998],\n 'Quito' : [-0.1806532,-78.46783820000002],\n 'Riobamba' : [-1.6635508,-78.65464600000001],\n 'Salinas' : [-2.2233633,-80.958462],\n 'Santo Domingo' : [-0.2389045,-79.17742679999998]\n}\n\n# Defining a color dictionary\ncol={'A':'red','B':'blue','C':'green','D':'pink','E':'beige',\n     0:'red',1:'blue',2:'green',3:'darkblue',4:'pink',5:'beige',13:'black'}\n\n#\ndef add_city_map(name,typ):\n    folium.Marker(\n         location=list(map(add,store_locations.get(name),[(0.5-rd.random())/20,(0.5-rd.random())/20])),\n         icon=folium.Icon(color=col.get(typ), icon='cloud'),\n    ).add_to(map_Ecuador)\n\nmap_Ecuador=folium.Map(location=[-1.233333, -78.516667],zoom_start=7)\n\n# Enabling clustering (also replace map_ecuador by store_cluster in the add_city_map function)\n# from folium.plugins import MarkerCluster\n#store_cluster=MarkerCluster().add_to(map_Ecuador)\n\n[add_city_map(x,y) for x,y in zip(pred_by_city1.city,pred_by_city1.type)]\nmap_Ecuador","metadata":{},"execution_count":32}],"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","mimetype":"text/x-python","version":"3.6.4","codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","pygments_lexer":"ipython3","nbconvert_exporter":"python"}},"nbformat":4}