{"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"8c41a03c-94e7-41b2-1da6-6886e21cbb92"},"source":""},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"fc21b28d-ac72-e741-6fb2-f2acacab6b35"},"outputs":[],"source":"# Bimbo Python XGBoost 5 week Lag Python script adapted from R\n# Rodolfo Lomascolo e-mail: r.lomascolo@rodolfolomascolo.com\n# Adapted from code by Bohdan Pavlyshenko  http://tinyurl.com/jd6k2kr\n# All credits for Bohdan Pavlyshenko and all bugs to me.\n#\n# Due to heavy usage of memory, this script will not run in kaggle.\n# the script runs only with a minimum of 35GB of physical or physical+swap memory\n# it can take a couple of hours to train depending on your machine, disk and memory\n# I used an 8 core, 56GB of RAM Azure DS13 machine with additional 100 GB of swap\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom subprocess import check_output\nfrom sklearn.metrics import make_scorer, mean_squared_error\nfrom sklearn.cross_validation import train_test_split\nfrom sklearn import preprocessing\nimport xgboost as xgb\nfrom sklearn import cross_validation\nimport scipy.stats as stats\nimport gc\n\n\ntrain = pd.read_csv('../input/train.csv') \ntest = pd.read_csv('../input/test.csv')\n\nprint('Train and Test Read')\n\ntrain['target'] = train['Demanda_uni_equil']\ntrain.drop(['Demanda_uni_equil'],axis=1, inplace = True)\n\ntrain['tst'] = 0\ntest['tst'] = 1\n\ndata = pd.concat([train,test], axis=0, copy=True)\n\nprint('Train and Test Concat')\ndel train\ndel test\ngc.collect()\n\nfor i in range(1,6):\n    lag = 'Lag' + str(i)\n    print('Lag:',lag)\n    \n    data1 = data[['Semana','Cliente_ID','Producto_ID','target']]\n    data1.loc[:,'Semana'] = data1['Semana'] +i\n    data1 = pd.groupby(data1,['Semana','Cliente_ID','Producto_ID']).mean() \n    data1 = data1.reset_index()\n    data1.rename(columns={'target': lag}, inplace=True)\n    data = pd.merge(data,data1,\n                    how='left',\n                    left_on=['Semana','Cliente_ID','Producto_ID'], \n                    right_on=['Semana','Cliente_ID','Producto_ID'],\n                    left_index=False, right_index=False, sort=True,\n                    suffixes=('_x', '_y'), copy=False, )\n    del data1\n    gc.collect()\n\ndata['TotalLags'] = data['Lag1'] + data['Lag2']+ data['Lag3']+ data['Lag4']+ data['Lag5']\n    \ndata=data[data['Semana']>8]  # NOW I WORK WITH WEEKS 9,10,11\n\nnAgencia = pd.DataFrame(pd.groupby(data,['Agencia_ID','Semana'])['target'].count())\nnAgencia = nAgencia.reset_index()\nnAgencia.rename(columns={'target': 'nAgencia'}, inplace=True)\nnAgencia = pd.DataFrame(pd.groupby(nAgencia,['Agencia_ID'])['nAgencia'].mean())\nnAgencia = nAgencia.reset_index()\n \n\ndata = pd.merge(data, nAgencia, \n                            how='left',\n                            left_on=['Agencia_ID'], \n                            right_on=['Agencia_ID'],\n                            left_index=False, right_index=False, sort=True,\n                            suffixes=('_x', '_y'), copy=False) \n\ndel nAgencia\ngc.collect()\nprint('merge completo nAgencia')\nprint(data.shape[0])\n\nnRuta_SAK = pd.DataFrame(pd.groupby(data,['Ruta_SAK','Semana'])['target'].count())\nnRuta_SAK = nRuta_SAK.reset_index()\nnRuta_SAK.rename(columns={'target': 'nRuta_SAK'}, inplace=True)\nnRuta_SAK = pd.DataFrame(pd.groupby(nRuta_SAK,['Ruta_SAK'])['nRuta_SAK'].mean())\nnRuta_SAK = nRuta_SAK.reset_index()\n \n\ndata = pd.merge(data, nRuta_SAK, \n                            how='left',\n                            left_on=['Ruta_SAK'], \n                            right_on=['Ruta_SAK'],\n                            left_index=False, right_index=False, sort=True,\n                            suffixes=('_x', '_y'), copy=False) \n\ndel nRuta_SAK\ngc.collect()\nprint('merge completo nRuta_SAK')\nprint(data.shape[0])\n\nnCliente_ID = pd.DataFrame(pd.groupby(data,['Cliente_ID','Semana'])['target'].count())\nnCliente_ID = nCliente_ID.reset_index()\nnCliente_ID.rename(columns={'target': 'nCliente_ID'}, inplace=True)\nnCliente_ID = pd.DataFrame(pd.groupby(nCliente_ID,['Cliente_ID'])['nCliente_ID'].mean())\nnCliente_ID = nCliente_ID.reset_index()\n \n\ndata = pd.merge(data, nCliente_ID, \n                            how='left',\n                            left_on=['Cliente_ID'], \n                            right_on=['Cliente_ID'],\n                            left_index=False, right_index=False, sort=True,\n                            suffixes=('_x', '_y'), copy=False) \n\ndel nCliente_ID\ngc.collect()\nprint('merge completo nCliente_ID')\nprint(data.shape[0])\n\nnProducto_ID = pd.DataFrame(pd.groupby(data,['Producto_ID','Semana'])['target'].count())\nnProducto_ID = nProducto_ID.reset_index()\nnProducto_ID.rename(columns={'target': 'nProducto_ID'}, inplace=True)\nnProducto_ID = pd.DataFrame(pd.groupby(nProducto_ID,['Producto_ID'])['nProducto_ID'].mean())\nnProducto_ID = nProducto_ID.reset_index()\n \n\ndata = pd.merge(data, nProducto_ID, \n                            how='left',\n                            left_on=['Producto_ID'], \n                            right_on=['Producto_ID'],\n                            left_index=False, right_index=False, sort=True,\n                            suffixes=('_x', '_y'), copy=False) \n\ndel nProducto_ID\ngc.collect()\nprint('merge completo nProducto_ID')\nprint(data.shape[0])\n\ndata.replace(np.nan,0, inplace=True)\n\ntrain = data[data['tst']==0]\npredict = data[data['tst']==1]\n\ntrain['target'] = np.log(train['target'] + 1)\n#train2 = train.sample(n=1000000)   <-- another possible reduction of data for fast testing\ntrain2 = train\ny = train['target']\nX = train[[  'Agencia_ID','Canal_ID','Cliente_ID','Producto_ID','Ruta_SAK',\n             'Lag1','Lag2','Lag3','Lag4','Lag5','TotalLags',\n             'nAgencia','nRuta_SAK','nCliente_ID','nProducto_ID']]\n\nprint(X.shape, y.shape)\n\n# split into train and test\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.01, random_state=1729)\nprint(X_train.shape, X_test.shape)\n\n\nxlf = xgb.XGBRegressor(max_depth=10, \n                        learning_rate=0.1, \n                        n_estimators=10, \n                        silent=True, \n                        objective='reg:linear', \n                        nthread=-1, \n                        gamma=0,\n                        min_child_weight=1, \n                        max_delta_step=0, \n                        subsample=0.85, \n                        colsample_bytree=0.7, \n                        colsample_bylevel=1, \n                        reg_alpha=0, \n                        reg_lambda=1, \n                        scale_pos_weight=1, \n                        seed=1440, \n                        missing=None)\n\nxlf.fit(X_train, y_train, eval_metric='rmse', verbose = True, eval_set = [(X_test, y_test)],early_stopping_rounds=100)\n\n# calculate the auc score\npreds = xlf.predict(X_test)\n\nprint('\\nRoot Mean Square error of Log(Demanda) X_test\" ', mean_squared_error(y_test,preds)**0.5)\n\n\n# GENERATE PREDICTION FOR 10-S11 FOR KAGGLE SUBMISSION\n\nprint('Make prediction for the 10th week')\ndata_test1=predict[predict['Semana']==10]\nids_10 =data_test1['id'] \ndata_test1 = data_test1[['Agencia_ID','Canal_ID','Cliente_ID','Producto_ID','Ruta_SAK',\n                         'Lag1','Lag2','Lag3','Lag4','Lag5','TotalLags',\n                         'nAgencia','nRuta_SAK','nCliente_ID','nProducto_ID']] \n\npred = xlf.predict(data_test1)\nres=np.exp(pred)-1\n\n# Create lagged values of target variable which will be used as a feature for the 11th week prediction \nprint('Create lagged values of target variable which will be used as a feature for the 11th week prediction')\ndata_test_lag1=data_test1[['Cliente_ID','Producto_ID']]\ndata_test_lag1['targetl1']=res\ndata_test_lag1 = pd.groupby(data_test_lag1,['Cliente_ID','Producto_ID']).mean() \ndata_test_lag1 = data_test_lag1.reset_index()\ndata_test_lag1.rename(columns={'targetl1': 'Lag1'}, inplace=True)\n\n# Save the predictions for first week\ndata_test1['Demanda_uni_equil'] = res\ndata_test1['id'] = ids_10.astype(int).tolist()\n\n# Make prediction for the 11th week\nprint('Make prediction for the 11th week')\n\ndata_test2=predict[predict['Semana']==11]\nids_11 =data_test2['id']\ndata_test2 = data_test2[[  'Agencia_ID','Canal_ID','Cliente_ID','Producto_ID','Ruta_SAK',\n             'Lag2','Lag3','Lag4','Lag5','TotalLags',\n             'nAgencia','nRuta_SAK','nCliente_ID','nProducto_ID']] \n\ndata_test2 = pd.merge(data_test2, data_test_lag1, \n                            how='left',\n                            left_on=['Cliente_ID','Producto_ID'], \n                            right_on=['Cliente_ID','Producto_ID'],\n                            left_index=False, right_index=False, sort=True,\n                            suffixes=('_x', '_y'), copy=False)\n\ndata_test2['TotalLags'] = data_test2['Lag1'] + data_test2['Lag2']+ data_test2['Lag3']+ data_test2['Lag4']+ data_test2['Lag5']\n\ndata_test2 = data_test2[['Agencia_ID','Canal_ID','Cliente_ID','Producto_ID','Ruta_SAK',\n                         'Lag1','Lag2','Lag3','Lag4','Lag5','TotalLags',\n                         'nAgencia','nRuta_SAK','nCliente_ID','nProducto_ID']] \n\ndata_test2.replace(np.nan,0, inplace=True)\npred = xlf.predict(data_test2)\nres=np.exp(pred)-1\n\n# Save the predictions for Second week\ndata_test2['Demanda_uni_equil'] = res\ndata_test2['id'] = ids_11.astype(int).tolist()\n\n\nsubmit = pd.concat([data_test1[['id','Demanda_uni_equil']],data_test2[['id','Demanda_uni_equil']]], axis=0, copy=True)\n\nprint('Saving Predictions file')\nsubmit.to_csv('Predictions/S10-S11-Prediction.csv', index=False)"}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}