{"nbformat":4,"nbformat_minor":1,"cells":[{"cell_type":"code","execution_count":null,"source":"from keras.models import Sequential\nfrom keras.layers import Dense, Dropout\nimport pandas as pd\nimport numpy as np\n\ndf_oil = pd.read_csv('../input/oil.csv',header=0)\ndf_store = pd.read_csv('../input/stores.csv',header=0)\ndf_item = pd.read_csv('../input/items.csv', header=0) \nall_items = pd.read_csv('../input/train.csv',header = 0)\ntst = pd.read_csv('../input/test.csv',header=0)\n\ndef getFormattedInput(it):\n    df_X = pd.DataFrame(pd.np.empty((len(it), 5)) * pd.np.nan,columns=['month','date','day','store_nbr','item_nbr'])\n    \n    it['onpromotion']=it['onpromotion'].fillna(0)\n\n    date = pd.to_datetime(it['date'],infer_datetime_format=True) # series object\n    temp_dict = {'month':date.dt.month,'date':date.dt.day,'day':date.dt.dayofweek}\n    df_X = pd.DataFrame(temp_dict)\n    df_X = pd.concat([df_X,it['store_nbr'],it['item_nbr']],axis=1)\n  \n    df_X = pd.get_dummies(df_X)\n    df_X = pd.concat([df_X, it['onpromotion']], axis=1)\n\n    return df_X\n\nitem_nbrs = df_item.values[0:-1,0]\n\ngroup_size = 10\n\nfor i in range(0,len(item_nbrs),group_size):\n    print('-----------------\\n',i,'\\n---------------\\n')\n    temp_item_nbrs = item_nbrs[i:(i+group_size)]\n    it = pd.DataFrame(columns=['id','date','store_nbr','item_nbr','unit_sales','onpromotion'])\n    \n    for j in range(0,len(temp_item_nbrs)):\n        itt = all_items.loc[(all_items['item_nbr']==int(temp_item_nbrs[j]))]\n        it = it.append(itt)\n        \n    len_it = len(it)  \n    it_max = it.nlargest(int(np.around(len_it/100)),'unit_sales')\n    it = it.drop(it_max.index.get_level_values(0))\n    it_min = it.nsmallest(int(np.around(len_it/200)),'unit_sales')\n    it = it.drop(it_min.index.get_level_values(0))\n    \n    Y = np.reshape(it['unit_sales'].values,[-1,1])\n\n    all_sample= np.arange(len(Y))\n    val_sample = np.random.randint(len(Y),size=int(np.around(len(Y)/20)))\n    train_sample = np.setdiff1d(all_sample,val_sample)\n\n    del it['unit_sales']\n    for j in range(0,group_size):\n        itt = tst.loc[(tst['item_nbr']==int(temp_item_nbrs[j]))]\n        it = it.append(itt)\n    \n    test_sample = np.arange(len(Y),len(it))\n    \n    df_X = getFormattedInput(it)\n    X = df_X.values\n    \n    tr_X = X[train_sample,:]\n    val_X = X[val_sample,:]\n    tr_Y = Y[train_sample]\n    val_Y = Y[val_sample]\n    test_X = X[test_sample,:]\n    \n    model = Sequential()\n    model.add(Dense(units=120,activation='relu',bias_initializer='ones',input_dim=X.shape[1]))\n    model.add(Dense(units=60,activation='relu'))\n    model.add(Dense(units=50,activation='relu'))\n    model.add(Dense(units=40,activation='relu'))\n    model.add(Dense(units=30,activation='relu'))\n    model.add(Dense(units=20,activation='relu'))\n    model.add(Dense(units=10,activation='relu'))\n    model.add(Dense(units=1, activation='linear'))\n\n    model.compile(loss='mean_squared_error', optimizer='adam', metrics=['accuracy'])\n\n    model.fit(tr_X, tr_Y, validation_data=(val_X, val_Y), epochs=100, batch_size=10000)\n\n    Y_hat = model.predict(val_X)\n\n    min_val = np.min(val_Y)\n\n    for j in range(0,len(Y_hat)):\n        if Y_hat[j]<min_val:\n            Y_hat[j]=min_val\n\n    print('NWRMSLE = ',np.dot((np.log(Y_hat+1)-np.log(val_Y+1)).T,(np.log(Y_hat+1)-np.log(val_Y+1))))\n    \n    Y_hat_test = model.predict(test_X)\n\n    for j in range(0,len(Y_hat_test)):\n        if Y_hat_test[j]<min_val:\n            Y_hat_test[j]=min_val\n    \n    print(len(Y_hat_test))\n    \n    temp = it['id'].values\n    ids = np.reshape(temp[test_sample],[-1,1])\n    result_array = np.concatenate((ids,Y_hat_test), axis=1)\n    result = pd.DataFrame(result_array, columns=['id','unit_sales'])\n\n    result.to_csv(str(i)+'.csv',index=False)\n\ndf_subm = pd.read_csv('sample_submission.csv',header=0)\ndf_subm['unit_sales']=0\n\nfor i in range(0,len(item_nbrs),group_size):\n    df_result = pd.read_csv(str(i)+'.csv',header=0)\n    \n    df_subm = pd.merge(df_subm, df_result, on='id', how='left')\n    df_subm = df_subm.fillna(0)\n    df_subm['unit_sales'] = df_subm['unit_sales_x']+df_subm['unit_sales_y']\n    del df_subm['unit_sales_x']\n    del df_subm['unit_sales_y']\n\ndf_subm.to_csv('final_submission.csv',index=False)","outputs":[],"metadata":{"_cell_guid":"1a620144-2be9-49e3-a6ba-7ff1abcd9d0d","_uuid":"cf8a9646c6db5bd3248a2daa10bd9982d114c037"}}],"metadata":{"kernelspec":{"display_name":"Python 3","name":"python3","language":"python"},"language_info":{"pygments_lexer":"ipython3","version":"3.6.3","name":"python","file_extension":".py","nbconvert_exporter":"python","codemirror_mode":{"version":3,"name":"ipython"},"mimetype":"text/x-python"}}}