{"cells":[{"metadata":{"_uuid":"a418fd0fb8aa0082eb1733a2e651a2b0e98ea458"},"cell_type":"markdown","source":"# Fastai based Mixed Input Model\nMy Idea here was to use the approach that succeded in the Rossman Kaggle challenge [see the fastai notebook here](https://github.com/fastai/fastai/blob/master/courses/dl1/lesson3-rossman.ipynb)\nThe data loading and initial feature engineering I borrowed heavily from [YouHan Lees Kernel](https://www.kaggle.com/youhanlee/hello-elo-ensemble-will-help-you) and [Hyun woo kim](https://www.kaggle.com/chocozzz/simple-data-exploration-with-python-lb-3-764/notebook)\nI think in this challenge the key thing is the feature engineering.\nHere I start with the model and use the borrowed feature engineering as a starting point.\nI might use these embedding layers to do the feature engineering on the new and historic transactions but as of Yet I'm not sure how to do it.\nSuggestions are always welcome\n"},{"metadata":{"trusted":true,"_uuid":"272d430596a11b125241b7253c8db09755052576"},"cell_type":"code","source":"%matplotlib inline\n%reload_ext autoreload\n%autoreload 2\nfrom fastai.structured import *\nfrom fastai.column_data import *","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"path='../input/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00c46ef3366002091c18c9084ef4f8b0aeaaa7b3"},"cell_type":"code","source":"new_transactions = pd.read_csv(f'{path}new_merchant_transactions.csv')\nnew_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d3551386a1529cd9cd8d025836a3454e3e2db16"},"cell_type":"code","source":"new_transactions['authorized_flag'] = new_transactions['authorized_flag']=='Y'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a524cd24bf3fc1c50cf58c3972f1b212fa13642"},"cell_type":"code","source":"def aggregate_new_transactions(new_trans):    \n    agg_func = {\n        'authorized_flag': ['sum', 'mean'],\n        'merchant_id': ['nunique'],\n        'city_id': ['nunique'],\n        'purchase_amount': ['sum', 'median', 'max', 'min', 'std'],\n        'installments': ['sum', 'median', 'max', 'min', 'std'],\n        'month_lag': ['min', 'max']\n        }\n    agg_new_trans = new_trans.groupby(['card_id']).agg(agg_func)\n    agg_new_trans.columns = ['new_' + '_'.join(col).strip() \n                           for col in agg_new_trans.columns.values]\n    agg_new_trans.reset_index(inplace=True)\n    \n    df = (new_trans.groupby('card_id')\n          .size()\n          .reset_index(name='new_transactions_count'))\n    \n    agg_new_trans = pd.merge(df, agg_new_trans, on='card_id', how='left')\n    \n    return agg_new_trans\n\nnew_trans = aggregate_new_transactions(new_transactions)\nnew_trans.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a524cd24bf3fc1c50cf58c3972f1b212fa13642"},"cell_type":"code","source":"del new_transactions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d98e1277cad1d8d7cbf4fbce88d7567010cc56b3"},"cell_type":"code","source":"historical_transactions = pd.read_csv('../input/historical_transactions.csv')\nhistorical_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11f02fc8752e241850e0564d46d9901cc1458632"},"cell_type":"code","source":"historical_transactions['authorized_flag'] = historical_transactions['authorized_flag']=='Y'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f5d75bc3d47dab9d1f30bc261e1193ff670f8cf"},"cell_type":"code","source":"def aggregate_historical_transactions(history):\n    \n    history.loc[:, 'purchase_date'] = pd.DatetimeIndex(history['purchase_date']).\\\n                                      astype(np.int64) * 1e-9\n    \n    agg_func = {\n        'authorized_flag': ['sum', 'mean'],\n        'merchant_id': ['nunique'],\n        'city_id': ['nunique'],\n        'purchase_amount': ['sum', 'median', 'max', 'min', 'std'],\n        'installments': ['sum', 'median', 'max', 'min', 'std'],\n        'purchase_date': [np.ptp],\n        'month_lag': ['min', 'max']\n        }\n    agg_history = history.groupby(['card_id']).agg(agg_func)\n    agg_history.columns = ['hist_' + '_'.join(col).strip() \n                           for col in agg_history.columns.values]\n    agg_history.reset_index(inplace=True)\n    \n    df = (history.groupby('card_id')\n          .size()\n          .reset_index(name='hist_transactions_count'))\n    \n    agg_history = pd.merge(df, agg_history, on='card_id', how='left')\n    \n    return agg_history\n\nhistory = aggregate_historical_transactions(historical_transactions)\nhistory.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f5d75bc3d47dab9d1f30bc261e1193ff670f8cf"},"cell_type":"code","source":"del historical_transactions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7cde0aaa9db4eb265f626510e7415e78d051c962"},"cell_type":"code","source":"def read_data(input_file):\n    df = pd.read_csv(input_file)\n    add_datepart(df,'first_active_month')\n    return df\ntrain = read_data('../input/train.csv')\ntest = read_data('../input/test.csv')\n\ntarget='target'\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"863b5a324fc6944f95508d1ce900ea6037e7fbc3"},"cell_type":"code","source":"train = pd.merge(train, history, on='card_id', how='left')\ntest = pd.merge(test, history, on='card_id', how='left')\n\ntrain = pd.merge(train, new_trans, on='card_id', how='left')\ntest = pd.merge(test, new_trans, on='card_id', how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"22167f5a0795b1c7aa51e9e972421956f84bdca8"},"cell_type":"code","source":"train=train.set_index('card_id')\ntest=test.set_index('card_id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05ebac39e6e97e81c1218820c1f80c447b306817"},"cell_type":"code","source":"cat_vars = [col  for col in train.columns if('feature' in col or 'first_active_month' in col)]\ncontin_vars = [col  for col in train.columns if ('feature' not in col and 'first_active_month' not in col) and target not in col]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e0051d1c6ca5604d2dc0a505a9fae627a37750a"},"cell_type":"code","source":"train = train[cat_vars+contin_vars+[target]].copy()\nn = len(train); n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2f4d90fbfc83bd63b3c6d45c9f8053d9d26bccf"},"cell_type":"code","source":"test[target] = 0\ntest = test[cat_vars+contin_vars+[target]].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85a1ce6f2b320728f7d5990c62226856cf6a5497"},"cell_type":"code","source":"for v in cat_vars: train[v] = train[v].astype('category').cat.as_ordered()\napply_cats(test, train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a505d3780ecc69b71fea1b05847f301fe57d18ce"},"cell_type":"code","source":"for v in contin_vars:\n    train[v] = train[v].fillna(0).astype('float32')\n    test[v] = test[v].fillna(0).astype('float32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b301b16938b07d50a9de17d2fd64f35c54f2cae5"},"cell_type":"code","source":"df, y, nas, mapper = proc_df(train, target, do_scale=True)\nn=len(df);n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d605c88c6f03c3c0c771654e870acac4e17e4160"},"cell_type":"code","source":"df_test, _, nas, mapper = proc_df(test, target, do_scale=True, \n                                  mapper=mapper, na_dict=nas)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a4fe610aa2ffdcae19b9d5abfa4f528c871c05b4"},"cell_type":"code","source":"train_ratio = 0.75\n# train_ratio = 0.9\ntrain_size = int(n * train_ratio); train_size\nval_idx = list(range(train_size, len(df)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b1697746ab6fdb929913152f2a322b17714f95d3"},"cell_type":"code","source":"model_data = ColumnarModelData.from_data_frame('.', val_idx, df, y.astype(np.float32), cat_flds=cat_vars, bs=64,\n                                       test_df=df_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf28c07adda2a02e1c556d508f79266f70288aa8"},"cell_type":"code","source":"cat_sz = [(c, len(train[c].cat.categories)+1) for c in cat_vars]\nemb_szs = [(c, min(50, (c+1)//2)) for _,c in cat_sz]\ny_range=(np.min(y),np.max(y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ce168111cb9864f2775970d4c784e152e9436bf"},"cell_type":"code","source":"def rmse(y_pred, targ):\n    var = np.square(targ - y_pred)\n    return math.sqrt(var.mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18ed27fa87f66932676f86720e068970bb077f7b"},"cell_type":"code","source":"model_data.get_learner??","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9179ed266822b89f0e56ca29f0edba00295c474d"},"cell_type":"code","source":"learn = model_data.get_learner(emb_szs, len(df.columns)-len(cat_vars),\n                   0.20, 1, [1000,500], [0.2,0.2], y_range=y_range,metrics=[rmse])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b187db917f6cffd1f7451b395ab369f7e267f0d"},"cell_type":"code","source":"learn.lr_find()\nlearn.sched.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9179ed266822b89f0e56ca29f0edba00295c474d"},"cell_type":"code","source":"learn.fit(1e-3, 1,cycle_len=5,use_clr_beta=(10,10,0.95,0.85))\nlearn.sched.plot_loss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9179ed266822b89f0e56ca29f0edba00295c474d","scrolled":true},"cell_type":"code","source":"learn.sched.plot_lr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76eddb319e8dd2b98573a3acd0fc6879eca4d152"},"cell_type":"code","source":"x,y=learn.predict_with_targs()\nrmse(x,y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5be999a47c9dad676eab234cd42342fc68b91dc4"},"cell_type":"code","source":"preds=learn.predict(is_test=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"402b43b612b2cab9e7b0041feba13bd91d36662e"},"cell_type":"code","source":"submission=pd.read_csv(f'{path}/sample_submission.csv',index_col='card_id')\nsubmission.loc[df_test.index]=preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"632eccb425e7d52c1f0bfa23794512f804480f43"},"cell_type":"code","source":"submission.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f001290bf7801ac80fb92d126f32b82e8715d30b"},"cell_type":"code","source":"!head submission.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d9e51d676e3575f8d55c7e1680cd422197c7ef30"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}