{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn import *\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"666ae3b789b4fee9091eaa3f681e1e42e569decc"},"cell_type":"code","source":"from fastai.tabular import *","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c3011b333c81a0347d7aeec844140ec76afb448f"},"cell_type":"markdown","source":"The train_df & test_df is borrowed from this wonderful [kernel](https://www.kaggle.com/gpreda/elo-world-high-score-without-blending)  by Gabriel. Thank you very much. The fastai library used here is from course V3. This complete kernel is heavily copied from ahem.. inspired from [here](https://github.com/fastai/course-v3/blob/master/nbs/dl1/lesson6-rossmann.ipynb). Special thanks to Jeremy for the wonderful course :)"},{"metadata":{"trusted":true,"_uuid":"d6a2e8229d9859e5b47509c5eeba68253223fedf"},"cell_type":"code","source":"train_df = pd.read_csv('../input/elo-world-high-score-without-blending/train_df.gz', compression='gzip', header=0,\n                      sep=',',quotechar='\"')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9db6aae4fd0081ee15da82ec563763c69d770faf"},"cell_type":"code","source":"test_df = pd.read_csv('../input/elo-world-high-score-without-blending/test_df.gz', compression='gzip', header=0,\n                      sep=',',quotechar='\"')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8dcb07b3c1354722893e9a3fdce90d0a1e4d77b8"},"cell_type":"code","source":"train_df.shape, test_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"553fe3c1fe42a7209559db9b02f0bf22d6165393"},"cell_type":"code","source":"df_indep = train_df.drop('target',axis=1)\nn_valid = 40000 #~20% of the training set\nn_trn = len(train_df)-n_valid","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb4d12f6393538fc970433779df115a67018c558"},"cell_type":"code","source":"n_trn, n_valid","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"af7cedbce1c87204047e91e33c4431c09619e405"},"cell_type":"markdown","source":"We add embeddings for all the categorical fields. "},{"metadata":{"trusted":true,"_uuid":"ee2318bdbe556129eac5027fcc4cef1090516305"},"cell_type":"code","source":"cat_flds = [n for n in df_indep.columns if train_df[n].nunique()<50 and n != 'outliers']\n','.join(cat_flds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f1930588244ca7477a48d17450277982f938146d"},"cell_type":"code","source":"len(cat_flds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"61646314d552a4d7badbc9ea32280b4267ee70c2"},"cell_type":"code","source":"for df in [train_df, df_indep, test_df]:\n    df.drop('first_active_month', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e437ce3cb639841e3e88abed613427a10a97b723"},"cell_type":"code","source":"for df in [train_df, df_indep, test_df]:\n    for n in cat_flds: \n        df[n] = df[n].astype('category').cat.as_ordered()\n    df['card_id'] = df['card_id'].astype('category').cat.as_ordered()\n    df['card_id_code'] = df.card_id.cat.codes\n    df.drop('card_id', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a772724a0c2ca5e37ce3069db599489e3b3d79ca"},"cell_type":"code","source":"train_df.shape, test_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"57b2aa41a2a505be2134220b1b1ad623ffd59257"},"cell_type":"code","source":"cont_flds = [n for n in df_indep.columns if n not in cat_flds and n!= 'outliers']\n','.join(cont_flds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"029e94542b34fec53d8e21be2b57005908f503b3"},"cell_type":"code","source":"procs=[FillMissing, Categorify, Normalize] #self-explanatory - neural nets like normalised values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"207d06906e20d6d8bdc3643f19a522145f828cb8"},"cell_type":"code","source":"len(cont_flds), len(cat_flds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a5ee0e3795801d0d2bbacd74bdba8512dd3242b"},"cell_type":"code","source":"dep_var = 'target'\ndf = train_df[cat_flds + cont_flds + [dep_var]].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cac993f0189e1f7a890b79044ee42af2df5f404d"},"cell_type":"code","source":"df[dep_var].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44f335336b8820a19462af3f99f1821c389b8c8b"},"cell_type":"code","source":"path = Path('../input/') #we need to give some path - doesn't matter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7994e7282e37e4ae1722b3ca4d15c52a602e3c0f"},"cell_type":"code","source":"data = (TabularList.from_df(df, path=path, cat_names=cat_flds, cont_names=cont_flds, procs=procs)\n                   .split_by_idx(range(n_valid))\n                   .label_from_df(cols=dep_var, label_cls=FloatList, log=False)\n                   .databunch())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c69e967f706847390b65b8466b4b00fdc8dc8189"},"cell_type":"code","source":"min_y = np.min(train_df['target'])*1.2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db223d55f88d203cdd9c86f9b2e51ba523842774"},"cell_type":"code","source":"max_y = np.max(train_df['target'])*1.2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9c1236ea19dd838929bfbf11da4eb5dc2cd6637"},"cell_type":"code","source":"y_range = torch.tensor([min_y, max_y], device=defaults.device)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9c2e23530a194bce0e8c3ba610e70a9cb8cafe88"},"cell_type":"markdown","source":"We added 2 hidden layers of size 1000,500 with dropouts .001, .01 respectively. The size of embeddings is chosen at half of the unique categories capped at 50 I afaik."},{"metadata":{"trusted":true,"_uuid":"16d2a37b1f94fde43df17d0d7a2635c13b78c2a8"},"cell_type":"code","source":"learn = tabular_learner(data, layers=[1000,500], ps=[0.001,0.01], emb_drop=0.04, \n                        y_range=y_range, metrics=rmse)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6adb18b579fc59622e4f5f5a5c7b4b229d0237be"},"cell_type":"markdown","source":"You can see the embeddings & our added hidden input layers with dropout & batchnorm below:"},{"metadata":{"trusted":true,"_uuid":"17a478d418979083e16bb4c47b9b96a8bdafd3db"},"cell_type":"code","source":"learn.model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c56dd528a351c5e1ed48d215fe076fa2d15fbf2"},"cell_type":"code","source":"len(data.train_ds.cont_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f0bd30894fe9e3800e81365e1b5b6df6713f11d"},"cell_type":"code","source":"learn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a012b731efee6e5d8c46ab9ab2e00d868e5c5bb"},"cell_type":"code","source":"learn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d5e5dd462dd140f7f4166021cdc7b20b2951ae4b"},"cell_type":"markdown","source":"We picked 1e-2 because that's where the loss is converging fastest"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"53c6695ef250cad148df1d5df4b13640be4361f6"},"cell_type":"code","source":"learn.fit_one_cycle(5, 1e-2, wd=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8bcb34b4fa48b92afb5c38e0c6ffe1125d135c70"},"cell_type":"code","source":"learn.recorder.plot_losses()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d5e7601a1c873bcaba2fe8e563b69084e9a68456"},"cell_type":"markdown","source":"The loss can be consistent if we use stratified kfold instead of using first 40k rows."},{"metadata":{"trusted":true,"_uuid":"2e93525e25c55891dfa6da67fc38948a2b5bf055"},"cell_type":"code","source":"# learn.fit_one_cycle(5, 3e-4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"675e04024d3570f0c883b4b72e0b1cff90d20d7d"},"cell_type":"code","source":"# learn.fit_one_cycle(4, 5e-2, wd=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7525b961c55d4897c7e74c4e4dd9ec8edf73f705"},"cell_type":"code","source":"# learn.fit_one_cycle(3, 1e-2, wd=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ee1844636f0fd73ef5d983ee0e3fc51a6b86b979"},"cell_type":"code","source":"learn.predict(df.iloc[0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"554709e60430a5850ad2f240421255014b6da63d"},"cell_type":"markdown","source":"In my local machine I've used older fastai version and trained using dataset split by stratified kfold. Due to some local GPU problem I wasn't able to output the predictions to csv. I think this solution might work good if you blend with your existing solutions. Do let me know the comments if you found it useful. Also let me know if you need a pytorch only version of this kernel."},{"metadata":{"trusted":true,"_uuid":"d4ab3e5ee0afff005a69738c0305614324b36cdc"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}