{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import lightgbm as lgb\nimport seaborn as sns\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.linear_model import LinearRegression\nimport statsmodels.api as sm\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import MinMaxScaler\nfrom scipy.stats import hmean,skew,kurtosis\nimport math \nimport IPython.display as ipd\ndtype_dict={\"id\":np.uint32,\n \"store_nbr\":np.uint8,\n \"item_nbr\":np.uint32,\n \"unit_sales\":np.float16}\nitem_dtype_dict={\"item_nbr\":np.uint32,\n \"family\":np.object,\n \"class\":np.uint8,\n \"perishable\":np.uint8}\nstore_dtype_dict={\"store_nbr\":np.uint8,\n \"store_nbr\":np.uint8,\n \"city\":np.object,\n \"state\":np.object,\n \"type\":np.object,\n \"cluster\":np.uint8}","execution_count":2,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nos.listdir(\"../input\")","execution_count":3,"outputs":[{"output_type":"execute_result","execution_count":3,"data":{"text/plain":"['transactions.csv',\n 'train.csv',\n 'items.csv',\n 'stores.csv',\n 'sample_submission.csv',\n 'test.csv',\n 'oil.csv',\n 'holidays_events.csv']"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"items = pd.read_csv(\"../input/items.csv\")\nitems = items.iloc[:,[0,3]]\nitems.head()","execution_count":4,"outputs":[{"output_type":"execute_result","execution_count":4,"data":{"text/plain":" item_nbr perishable\n0 96995 0\n1 99197 0\n2 103501 0\n3 103520 0\n4 103665 1","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
item_nbrperishable
0969950
1991970
21035010
31035200
41036651
\n
"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\",dtype=dtype_dict, converters={'unit_sales': lambda u: np.log1p(float(u)) if float(u) > 0 else 0})\ntrain['Year'] = pd.DatetimeIndex(train['date']).year\ntrain_2017=train.loc[train['Year'] ==2017]\ndel train \nimport gc\ngc.collect()","execution_count":5,"outputs":[{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:1: ParserWarning: Both a converter and dtype were specified for column unit_sales - only the converter will be used\n \"\"\"Entry point for launching an IPython kernel.\n/opt/conda/lib/python3.6/site-packages/IPython/core/interactiveshell.py:2785: DtypeWarning: Columns (5) have mixed types. Specify dtype option on import or set low_memory=False.\n interactivity=interactivity, compiler=compiler, result=result)\n","name":"stderr"},{"output_type":"execute_result","execution_count":5,"data":{"text/plain":"21"},"metadata":{}}]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_promo_2017 = train_2017.set_index([\"store_nbr\", \"item_nbr\", \"date\"])[[\"onpromotion\"]].unstack().fillna(0)\ntrain_sales_2017 = train_2017.set_index([\"store_nbr\", \"item_nbr\", \"date\"])[[\"unit_sales\"]].unstack().fillna(0)\ntrain_sales_2017.columns = train_sales_2017.columns.get_level_values(1)","execution_count":6,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":7,"outputs":[{"output_type":"execute_result","execution_count":7,"data":{"text/plain":" id date store_nbr ... Year Month Day\n101688779 101688779 2017-01-01 25 ... 2017 1 1\n101688780 101688780 2017-01-01 25 ... 2017 1 1\n101688781 101688781 2017-01-01 25 ... 2017 1 1\n101688782 101688782 2017-01-01 25 ... 2017 1 1\n101688783 101688783 2017-01-01 25 ... 2017 1 1\n\n[5 rows x 9 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
iddatestore_nbritem_nbrunit_salesonpromotionYearMonthDay
1016887791016887792017-01-0125991970.693147False201711
1016887801016887802017-01-01251036652.079442False201711
1016887811016887812017-01-01251055740.693147False201711
1016887821016887822017-01-01251058571.609438False201711
1016887831016887832017-01-01251067161.098612False201711
\n
"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv(\"../input/test.csv\")\ntest['date'] = pd.to_datetime(test['date'])\ntest_promo_2017 = test.set_index([\"store_nbr\", \"item_nbr\", \"date\"])[[\"onpromotion\"]].unstack().fillna(0)\ntest_promo_2017 = test_promo_2017.reindex(train_promo_2017.index).fillna(0)\ntest_train_promo_2017 = pd.concat([train_promo_2017,test_promo_2017],axis=1)\ntest_train_promo_2017.columns = test_train_promo_2017.columns.get_level_values(1)","execution_count":8,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"items = pd.read_csv(\"../input/items.csv\")\nitems = items.iloc[:,[0,2]]\ntrain_class_2017=pd.merge(train_2017,items,how = 'left',on='item_nbr')\ntrain_class_promo_2017 = train_class_2017.set_index([\"store_nbr\", \"item_nbr\",\"class\",\"date\"])[[\"onpromotion\"]].unstack().fillna(0)\ntest = pd.read_csv(\"../input/test.csv\")\ntest['date'] = pd.to_datetime(test['date'])\ntest_class_2017 = pd.merge(test,items,how = 'left',on='item_nbr')\ntest_class_promo_2017 = test_class_2017.set_index([\"store_nbr\", \"item_nbr\",\"class\", \"date\"])[[\"onpromotion\"]].unstack().fillna(0)\ntest_class_promo_2017 = test_class_promo_2017.reindex(train_class_promo_2017.index).fillna(0)\ntest_train_class_promo_2017 = pd.concat([train_class_promo_2017,test_class_promo_2017],axis=1)\ntest_train_class_promo_2017.columns = test_train_class_promo_2017.columns.get_level_values(1)\ntrain_class_sales_2017 = train_class_2017.set_index([\"store_nbr\", \"item_nbr\",\"class\",\"date\"])[[\"unit_sales\"]].unstack().fillna(0)\ntrain_class_sales_2017.columns = train_class_sales_2017.columns.get_level_values(1)\n","execution_count":9,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef features_dataset_val(dataframe,val_date):\n df_val = []\n y= []\n y = pd.DataFrame(y)\n df_val = pd.DataFrame(df_val)\n month = int(val_date.strftime('%m'))\n week = val_date.isocalendar()[1]\n for i in [10, 20, 30,40,50,60,120]:\n df_val['mean_for_-{}'.format(i)] = dataframe[pd.date_range(start =(val_date-timedelta(days=i)), periods=i, freq = 'D')].mean(axis=1)\n df_val['std_for_-{}'.format(i)] = dataframe[pd.date_range(start =(val_date-timedelta(days=i)), periods=i,freq = 'D')].std(axis=1)\n df_val['var_for_-{}'.format(i)] = dataframe[pd.date_range(start =(val_date-timedelta(days=i)), periods=i, freq = 'D')].var(axis=1)\n df_val['median_for_-{}'.format(i)] = dataframe[pd.date_range(start =(val_date-timedelta(days=i)),periods=i,freq = 'D')].median(axis=1)\n df_val['zero_sales_for_-{}'.format(i)] = dataframe[pd.date_range(start =(val_date-timedelta(days=i)), periods=i, freq = 'D')].apply(lambda x : list(x).count(0),axis=1)\n df_val['kurtosis_for_-{}'.format(i)] = dataframe[pd.date_range(start =(val_date-timedelta(days=i)), periods=i, freq = 'D')].apply(kurtosis, axis=1)\n df_val['hmean_for_-{}'.format(i)] = (dataframe[pd.date_range(start =(val_date-timedelta(days=i)), periods=i, freq = 'D')]+0.0001).apply(hmean, axis=1)\n df_val['skew_for_-{}'.format(i)] = dataframe[pd.date_range(start =(val_date-timedelta(days=i)), periods=i, freq = 'D')].apply(skew, axis=1)\n df_val['mean_for_class_-{}'.format(i)] = train_class_sales_2017[pd.date_range(start =(val_date-timedelta(days=i)), periods=i, freq = 'D')].mean(axis=1).values\n df_val['zero_sales_for_class_-{}'.format(i)] = train_class_sales_2017[pd.date_range(start =(val_date-timedelta(days=i)), periods=i, freq = 'D')].apply(lambda x : list(x).count(0),axis=1).values\n\n df_val['promosum{}'.format(16)] = test_train_promo_2017[pd.date_range(start =(val_date-timedelta(days=16)), periods=16, freq = 'D')].sum(axis=1)\n df_val['promosum{}'.format(30)] = test_train_promo_2017[pd.date_range(start =(val_date-timedelta(days=30)), periods=30, freq = 'D')].sum(axis=1)\n df_val['promosum{}'.format(60)] = test_train_promo_2017[pd.date_range(start =(val_date-timedelta(days=60)), periods=60, freq = 'D')].sum(axis=1)\n df_val['promosum_class{}'.format(16)] = test_train_class_promo_2017[pd.date_range(start =(val_date-timedelta(days=16)), periods=16, freq = 'D')].sum(axis=1).values\n df_val['promosum_class{}'.format(30)] = test_train_class_promo_2017[pd.date_range(start =(val_date-timedelta(days=30)), periods=30, freq = 'D')].sum(axis=1).values\n df_val['promosum_class{}'.format(60)] = test_train_class_promo_2017[pd.date_range(start =(val_date-timedelta(days=60)), periods=60, freq = 'D')].sum(axis=1).values\n \n for i in range(7): #one month worth of day of week values\n df_val['mean_for_dow_4_{}'.format(i)] = dataframe[pd.date_range(start =(val_date - timedelta(days=28-i)),periods=4,freq='7D')].mean(axis=1)\n df_val['mean_for_dow_8_{}'.format(i)] = dataframe[pd.date_range(start =(val_date - timedelta(days=56-i)),periods=8,freq='7D')].mean(axis=1)\n df_val['mean_for_dow_16_{}'.format(i)] = dataframe[pd.date_range(start =(val_date - timedelta(days=112-i)),periods=16,freq='7D')].mean(axis=1)\n df_val['mean_for_dow_20_{}'.format(i)] = dataframe[pd.date_range(start =(val_date - timedelta(days=140-i)),periods=20,freq='7D')].mean(axis=1)\n df_val['zero_sales_for_dow_4_{}'.format(i)] = dataframe[pd.date_range(start =(val_date - timedelta(days=28-i)),periods=4,freq='7D')].apply(lambda x : list(x).count(0),axis=1)\n df_val['zero_sales_for_dow_8_{}'.format(i)] = dataframe[pd.date_range(start =(val_date - timedelta(days=56-i)),periods=8,freq='7D')].apply(lambda x : list(x).count(0),axis=1)\n df_val['zero_sales_for_dow_16_{}'.format(i)] = dataframe[pd.date_range(start =(val_date - timedelta(days=112-i)),periods=16,freq='7D')].apply(lambda x : list(x).count(0),axis=1)\n df_val['zero_sales_for_dow_20_{}'.format(i)] = dataframe[pd.date_range(start =(val_date - timedelta(days=140-i)),periods=20,freq='7D')].apply(lambda x : list(x).count(0),axis=1)\n\n for i in range(16):\n y['day{}'.format(i)] = dataframe[(val_date+timedelta(days=i))]\n df_val['promo_{}'.format(i)] = test_train_promo_2017[val_date+timedelta(days=i)].values\n return df_val,y","execution_count":10,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef features_dataset_test(dataframe,test_date):\n # find mean,median,std,var of data for the specific store and item number that is a month before\n df_test = []\n df_test = pd.DataFrame(df_test)\n month = int(test_date.strftime('%m'))\n week = test_date.isocalendar()[1]\n for i in [10, 20, 30,40,50,60,120]:\n df_test['mean_for_-{}'.format(i)] = dataframe[pd.date_range(start =(test_date-timedelta(days=i)), periods=i, freq = 'D')].mean(axis=1)\n df_test['std_for_-{}'.format(i)] = dataframe[pd.date_range(start =(test_date-timedelta(days=i)), periods=i, freq = 'D')].std(axis=1)\n df_test['var_for_-{}'.format(i)] = dataframe[pd.date_range(start =(test_date-timedelta(days=i)), periods=i, freq = 'D')].var(axis=1)\n df_test['median_for_-{}'.format(i)] = dataframe[pd.date_range(start =(test_date-timedelta(days=i)), periods=i, freq = 'D')].median(axis=1)\n df_test['zero_sales_for_-{}'.format(i)] = dataframe[pd.date_range(start =(test_date-timedelta(days=i)), periods=i, freq = 'D')].apply(lambda x : list(x).count(0),axis=1)\n df_test['kurtosis_for_-{}'.format(i)] = dataframe[pd.date_range(start =(test_date-timedelta(days=i)), periods=i, freq = 'D')].apply(kurtosis, axis=1)\n df_test['hmean_for_-{}'.format(i)] = (dataframe[pd.date_range(start =(test_date-timedelta(days=i)), periods=i, freq = 'D')]+0.0001).apply(hmean, axis=1)\n df_test['skew_for_-{}'.format(i)] = dataframe[pd.date_range(start =(test_date-timedelta(days=i)), periods=i, freq = 'D')].apply(skew, axis=1)\n df_test['mean_for_class_-{}'.format(i)] = train_class_sales_2017[pd.date_range(start =(test_date-timedelta(days=i)), periods=i, freq = 'D')].mean(axis=1).values\n df_test['zero_sales_for_class_-{}'.format(i)] = train_class_sales_2017[pd.date_range(start =(test_date-timedelta(days=i)), periods=i, freq = 'D')].apply(lambda x : list(x).count(0),axis=1).values\n\n df_test['promosum{}'.format(16)] = test_train_promo_2017[pd.date_range(start =(test_date-timedelta(days=16)), periods=16, freq = 'D')].sum(axis=1)\n df_test['promosum{}'.format(30)] = test_train_promo_2017[pd.date_range(start =(test_date-timedelta(days=30)), periods=30, freq = 'D')].sum(axis=1)\n df_test['promosum{}'.format(60)] = test_train_promo_2017[pd.date_range(start =(test_date-timedelta(days=60)), periods=60, freq = 'D')].sum(axis=1)\n df_test['promosum_class{}'.format(16)] = test_train_class_promo_2017[pd.date_range(start =(test_date-timedelta(days=16)), periods=16, freq = 'D')].sum(axis=1).values\n df_test['promosum_class{}'.format(30)] = test_train_class_promo_2017[pd.date_range(start =(test_date-timedelta(days=30)), periods=30, freq = 'D')].sum(axis=1).values\n df_test['promosum_class{}'.format(60)] = test_train_class_promo_2017[pd.date_range(start =(test_date-timedelta(days=60)), periods=60, freq = 'D')].sum(axis=1).values\n \n \n for i in range(7): #one month worth of day of week values\n df_test['mean_for_dow4_{}'.format(i)] = dataframe[pd.date_range(start =(test_date - timedelta(days=28-i)),periods=4,freq='7D')].mean(axis=1)\n df_test['mean_for_dow8_{}'.format(i)] = dataframe[pd.date_range(start =(test_date - timedelta(days=56-i)),periods=8,freq='7D')].mean(axis=1)\n df_test['mean_for_dow16_{}'.format(i)] = dataframe[pd.date_range(start =(test_date - timedelta(days=112-i)),periods=16,freq='7D')].mean(axis=1)\n df_test['mean_for_dow20_{}'.format(i)] = dataframe[pd.date_range(start =(test_date - timedelta(days=140-i)),periods=20,freq='7D')].mean(axis=1)\n df_test['zero_sales_for_dow_4_{}'.format(i)] = dataframe[pd.date_range(start =(test_date - timedelta(days=28-i)),periods=4,freq='7D')].apply(lambda x : list(x).count(0),axis=1)\n df_test['zero_sales_for_dow_8_{}'.format(i)] = dataframe[pd.date_range(start =(test_date - timedelta(days=56-i)),periods=8,freq='7D')].apply(lambda x : list(x).count(0),axis=1)\n df_test['zero_sales_for_dow_16_{}'.format(i)] = dataframe[pd.date_range(start =(test_date - timedelta(days=112-i)),periods=16,freq='7D')].apply(lambda x : list(x).count(0),axis=1)\n df_test['zero_sales_for_dow_20_{}'.format(i)] = dataframe[pd.date_range(start =(test_date - timedelta(days=140-i)),periods=20,freq='7D')].apply(lambda x : list(x).count(0),axis=1)\n\n for i in range(16):\n df_test['promo{}'.format(i)] = test_train_promo_2017[test_date+timedelta(days=i)].values.astype(np.uint8)\n #df_test = pd.merge(test.loc[test['date']==test_date],df_test,on=(['item_nbr','store_nbr']),how = 'left').fillna(0)\n return df_test","execution_count":11,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# use mean,std,var of 30/10 days before to predict the unit_sales\ndef features_dataset_train(dataframe,train_date):\n \n df_train = []\n y= []\n y = pd.DataFrame(y)\n df_train = pd.DataFrame(df_train)\n month = int(train_date.strftime('%m'))\n week = train_date.isocalendar()[1]\n for i in [10, 20, 30,40,50,60,120]:\n df_train['mean_for_-{}'.format(i)] = dataframe[pd.date_range(start =(train_date-timedelta(days=i)), periods=i,freq = 'D')].mean(axis=1)\n df_train['std_for_-{}'.format(i)] = dataframe[pd.date_range(start =(train_date-timedelta(days=i)), periods=i, freq = 'D')].std(axis=1)\n df_train['var_for_-{}'.format(i)] = dataframe[pd.date_range(start =(train_date-timedelta(days=i)), periods=i, freq = 'D')].var(axis=1)\n df_train['median_for_-{}'.format(i)] = dataframe[pd.date_range(start =(train_date-timedelta(days=i)), periods=i, freq = 'D')].median(axis=1) \n df_train['zero_sales_for_-{}'.format(i)] = dataframe[pd.date_range(start =(train_date-timedelta(days=i)), periods=i, freq = 'D')].apply(lambda x : list(x).count(0),axis=1)\n df_train['kurtosis_for_-{}'.format(i)] = dataframe[pd.date_range(start =(train_date-timedelta(days=i)), periods=i, freq = 'D')].apply(kurtosis, axis=1)\n df_train['hmean_for_-{}'.format(i)] = (dataframe[pd.date_range(start =(train_date-timedelta(days=i)), periods=i, freq = 'D')]+0.0001).apply(hmean, axis=1)\n df_train['skew_for_-{}'.format(i)] = dataframe[pd.date_range(start =(train_date-timedelta(days=i)), periods=i, freq = 'D')].apply(skew, axis=1)\n df_train['mean_for_class_-{}'.format(i)] = train_class_sales_2017[pd.date_range(start =(train_date-timedelta(days=i)), periods=i, freq = 'D')].mean(axis=1).values\n df_train['zero_sales_for_class_-{}'.format(i)] = train_class_sales_2017[pd.date_range(start =(train_date-timedelta(days=i)), periods=i, freq = 'D')].apply(lambda x : list(x).count(0),axis=1).values\n \n \n df_train['promosum{}'.format(16)] = test_train_promo_2017[pd.date_range(start =(train_date-timedelta(days=16)), periods=16, freq = 'D')].sum(axis=1)\n df_train['promosum{}'.format(30)] = test_train_promo_2017[pd.date_range(start =(train_date-timedelta(days=30)), periods=30, freq = 'D')].sum(axis=1)\n df_train['promosum{}'.format(60)] = test_train_promo_2017[pd.date_range(start =(train_date-timedelta(days=60)), periods=60, freq = 'D')].sum(axis=1)\n df_train['promosum_class{}'.format(16)] = test_train_class_promo_2017[pd.date_range(start =(train_date-timedelta(days=16)), periods=16, freq = 'D')].sum(axis=1).values\n df_train['promosum_class{}'.format(30)] = test_train_class_promo_2017[pd.date_range(start =(train_date-timedelta(days=30)), periods=30, freq = 'D')].sum(axis=1).values\n df_train['promosum_class{}'.format(60)] = test_train_class_promo_2017[pd.date_range(start =(train_date-timedelta(days=60)), periods=60, freq = 'D')].sum(axis=1).values\n \n \n for i in range(7): #one month worth of day of week values\n df_train['mean_for_dow4_{}'.format(i)] = dataframe[pd.date_range(start =(train_date - timedelta(days=28-i)),periods=4,freq='7D')].mean(axis=1)\n df_train['mean_for_dow8_{}'.format(i)] = dataframe[pd.date_range(start =(train_date - timedelta(days=56-i)),periods=8,freq='7D')].mean(axis=1)\n df_train['mean_for_dow16_{}'.format(i)] = dataframe[pd.date_range(start =(train_date - timedelta(days=112-i)),periods=16,freq='7D')].mean(axis=1)\n df_train['mean_for_dow20_{}'.format(i)] = dataframe[pd.date_range(start =(train_date - timedelta(days=140-i)),periods=20,freq='7D')].mean(axis=1)\n df_train['zero_sales_for_dow_4_{}'.format(i)] = dataframe[pd.date_range(start =(train_date - timedelta(days=28-i)),periods=4,freq='7D')].apply(lambda x : list(x).count(0),axis=1)\n df_train['zero_sales_for_dow_8_{}'.format(i)] = dataframe[pd.date_range(start =(train_date - timedelta(days=56-i)),periods=8,freq='7D')].apply(lambda x : list(x).count(0),axis=1)\n df_train['zero_sales_for_dow_16_{}'.format(i)] = dataframe[pd.date_range(start =(train_date - timedelta(days=112-i)),periods=16,freq='7D')].apply(lambda x : list(x).count(0),axis=1)\n df_train['zero_sales_for_dow_20_{}'.format(i)] = dataframe[pd.date_range(start =(train_date - timedelta(days=140-i)),periods=20,freq='7D')].apply(lambda x : list(x).count(0),axis=1)\n\n for i in range(16):\n y['day{}'.format(i)] = dataframe[(train_date+timedelta(days=i))]\n df_train['promo_{}'.format(i)] = test_train_promo_2017[train_date+timedelta(days=i)].values\n return df_train,y","execution_count":12,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def find_weights(date):\n weights_tmp= []\n weights_tmp = pd.DataFrame(weights_tmp)\n weights_tmp = pd.merge(pd.DataFrame(train_sales_2017[date]).reset_index(['store_nbr','item_nbr']),items)['perishable'].fillna(0)\n return weights_tmp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import datetime\nfrom datetime import timedelta, date\ntest_date = date(2017,8,16)\ntest = pd.read_csv(\"../input/test.csv\")\ntest['date'] = pd.to_datetime(test['date'])\nX_test = []\nX_test = features_dataset_test(train_sales_2017,test_date)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import datetime\nfrom datetime import timedelta, date\ntrain_date_gbm = date(2017,7,26)\nX_tmp = []\nX_train_gbm = []\ny_tmp = []\ny_train_gbm = []\nfor i in range(8): #number represents the number of weeks. not sure whats the right number. when i increased from 4 to 6(more data) the score improved. \n X_tmp,y_tmp = features_dataset_train(train_sales_2017,train_date_gbm) \n X_train_gbm.append(X_tmp)\n y_train_gbm.append(y_tmp)\n train_date_gbm= train_date_gbm - timedelta(days=7)\n #print(X_train_gbm)\nX_train_gbm = pd.concat(X_train_gbm)\ny_train_gbm = pd.concat(y_train_gbm)\ntrain_date_gbm = date(2017,7,26)\nweight_tmp = []\nweight_gbm = []\nfor i in range(8): #just try with 10 first then increase/decrease accordingly up to u\n weight_tmp = find_weights(train_date_gbm)\n weight_gbm.append(weight_tmp)\n train_date_gbm= train_date_gbm - timedelta(days=7)\nweight_gbm = pd.concat(weight_gbm)\nval_date_gbm = date(2017,7,26)\nweight_val_gbm = []\nweight_tmp = find_weights(val_date_gbm)\nweight_val_gbm.append(weight_tmp)\nweight_val_gbm = pd.concat(weight_val_gbm)\nX_val_gbm = []\ny_val_gbm = []\nval_date_gbm = date(2017,7,26)\nX_val_gbm,y_val_gbm = features_dataset_val(train_sales_2017,val_date_gbm)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\ny_test_final = []\nparams={}\nparams['max_depth'] = 6\nparams['learning_rate'] = 0.01\nparams['boosting_type'] = 'gbdt'\nparams['objective'] = 'regression'\nparams['metric'] = 'mse' \nparams['bagging_fraction'] = 0.75 \nparams['feature_fraction'] = 0.75\nparams['num_leaves'] = 2**6-1 \nparams['min_data'] = 37 \nparams['verbose'] = 2\nparams['num_threads'] = 4\nfor i in range(16):\n print(\"Step %d\" % (i+1))\n lgb_train = lgb.Dataset(X_train_gbm, label=y_train_gbm.iloc[:,i],weight =np.array(weight_gbm*0.25+1))\n lgb_val = lgb.Dataset(X_val_gbm, label = y_val_gbm.iloc[:,i],reference = lgb_train,weight = np.array(weight_val_gbm*0.25+1))\n lgb_model = lgb.train(params,lgb_train,num_boost_round=2000,valid_sets=[lgb_train,lgb_val],early_stopping_rounds = 50,verbose_eval = 50)\n y_test = lgb_model.predict(X_test,num_iteration = lgb_model.best_iteration)\n y_test_final.append(y_test)\n print(lgb_model.feature_importance('gain'))\n print(y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_sales = pd.DataFrame(\nnp.array(y_test_final).transpose(), index=train_sales_2017.index,\n columns=pd.date_range(\"2017-08-16\", periods=16)).stack().reset_index()\nsubmission_sales = submission_sales.rename(columns = {'level_2' : 'date',0:'unit_sales'})\nsubmission = pd.merge(test,pd.DataFrame(submission_sales),on =['store_nbr','item_nbr','date'],how = 'left').fillna(0)\nsubmission = submission.drop(columns = {'date','store_nbr','item_nbr','onpromotion'})\nsubmission[\"unit_sales\"] = np.expm1(submission[\"unit_sales\"])\nsubmission.loc[submission['unit_sales']<0,['unit_sales']] = 0\nsubmission.to_csv(\"submission.csv\",index=None) #put own path","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}