{"cells":[{"metadata":{"_uuid":"59d9e40a960d7a8c9518f8e268e36d3c4fedb3b7"},"cell_type":"markdown","source":"# Work in progress"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.\n\n# My imports\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"84fc16e9f96a4dc9cce55469ad473b0d512559b2"},"cell_type":"code","source":"data_root = \"../input\"\noutput_dir = './'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c82271a015cf971ecb9bd0192173253a766471c1"},"cell_type":"code","source":"test = pd.read_csv(os.path.join(data_root, \"test.csv\"))\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv(os.path.join(data_root, \"train.csv\"))\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f4c084df04205ed4f3a669a7852b79f125f779a"},"cell_type":"code","source":"merchants = pd.read_csv(os.path.join(data_root, \"merchants.csv\"))\nmerchants.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a4ee1f3210f13f882e60b69f2cdb8a629c47125"},"cell_type":"code","source":"historical_transactions = pd.read_csv(os.path.join(data_root, \"historical_transactions.csv\"))\nhistorical_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a83c82faafd9898742872a14fdf1900d6adaae9"},"cell_type":"code","source":"new_merchant_transactions = pd.read_csv(os.path.join(data_root, \"new_merchant_transactions.csv\"))\nnew_merchant_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f6327ffe407addfc5ae9ede2b5e7a6a971f2446"},"cell_type":"code","source":"engineered_test = pd.DataFrame()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bb6984ffc4389d793b9ea4b9eee8a2bf95d7121"},"cell_type":"code","source":"engineered_test = test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c5f109dd17619d49ee219749561db2da07f3f928"},"cell_type":"code","source":"engineered_train = pd.DataFrame()\nengineered_train_outlier = pd.DataFrame()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"938ea1a56850ae19408ca10a02184ad325f0c676"},"cell_type":"markdown","source":"# Take a look at the so called training features"},{"metadata":{"trusted":true,"_uuid":"d877ab304540bb5fb68a7626661ace6c9f76ff77"},"cell_type":"code","source":"plt.figure()\ntrain['target'].hist()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9a8d8168750b7ec52ebc0fd2916b185b43c0efb"},"cell_type":"code","source":"train[train['target'] < -20]['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0a90149870d876f8f404c21b26f0ab99e5c08bc6"},"cell_type":"markdown","source":"Huh? Is this some kind of place holder for spam cards??\nMaybe a binary classifier should be used to determine this group"},{"metadata":{"trusted":true,"_uuid":"12265fee252183923af2bd0f82102261a6b76286"},"cell_type":"code","source":"engineered_train = train[train['target'] > -20]\nengineered_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88860cba12dc0516127a42a051da4f85e36a92fe"},"cell_type":"code","source":"engineered_train_outlier = train[train['target'] < -20]\nengineered_train_outlier.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dbca1d316175b1a48699d1a83d319c94a635d027"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 6, figsize=(24,4))\nengineered_train['feature_1'].value_counts().sort_index().plot(kind='bar', ax=ax[0], color='teal', title='train feature_1')\nengineered_train['feature_2'].value_counts().sort_index().plot(kind='bar', ax=ax[1], color='brown', title='train feature_2')\nengineered_train['feature_3'].value_counts().sort_index().plot(kind='bar', ax=ax[2], color='gold', title='train feature_3')\ntest['feature_1'].value_counts().sort_index().plot(kind='bar', ax=ax[3], color='teal', title='test feature_1')\ntest['feature_2'].value_counts().sort_index().plot(kind='bar', ax=ax[4], color='brown', title='test feature_2')\ntest['feature_3'].value_counts().sort_index().plot(kind='bar', ax=ax[5], color='gold', title='test feature_3')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"32206db77ba8a3bb5fee785cdcd90a5bb0c77542"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 6, figsize=(24,4))\nfor i in range(5):\n    title = \"feature_1=\" + str(i+1)\n    engineered_train[engineered_train['feature_1'] == i+1]['target'].hist(ax=ax[i], bins=50)\n    ax[i].set_title(title)\nengineered_train['target'].hist(ax=ax[-1], bins=50)\nax[-1].set_title('overall train target')\nfig.suptitle('Target distribution with all feature_1 values')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e19231e15b14e00515dd859b03f1428c2a81efaa"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 4, figsize=(20,4))\nfor i in range(3):\n    title = \"feature_2=\" + str(i+1)\n    engineered_train[engineered_train['feature_2'] == i+1]['target'].hist(ax=ax[i], bins=50)\n    ax[i].set_title(title)\nengineered_train['target'].hist(ax=ax[-1], bins=50)\nax[-1].set_title('overall train target')\nfig.suptitle('Target distribution with all feature_2 values')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e56a4ff5946b27a6c839526d38405e6a74835b9c"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 3, figsize=(12,4))\nfor i in range(2):\n    title = \"feature_3=\" + str(i)\n    engineered_train[engineered_train['feature_3'] == i]['target'].hist(ax=ax[i], bins=50)\n    ax[i].set_title(title)\nengineered_train['target'].hist(ax=ax[-1], bins=50)\nax[-1].set_title('overall train target')\nfig.suptitle('Target distribution with all feature_3 values')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a09065dda2111213c78d2ae78998dbfe8c00f088"},"cell_type":"markdown","source":"It seems that feature 1, 2, and 3 are pretty much useless\n\nThen how about active month?"},{"metadata":{"trusted":true,"_uuid":"9493022c4ee1ae44f249ba18035161f8d7d8b18e"},"cell_type":"code","source":"def month_to_timestamp(month):\n    time_format = '%Y-%m'\n    if type(month) is str:\n        timestamp = datetime.strptime(month,time_format).timestamp()\n        return timestamp\n    else:\n        return 1.5 * 1000000000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b68a948d2cb98748026b2bbeef35c33438421dee"},"cell_type":"code","source":"engineered_train['first_active_month_timestamp'] = train[train['target'] > -20]['first_active_month'].apply(month_to_timestamp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"646281887348903ab49351a40c1421ab340cb243"},"cell_type":"code","source":"engineered_train_outlier['first_active_month_timestamp'] = train[train['target'] < -20]['first_active_month'].apply(month_to_timestamp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"08641ab3c0e2f40be87acf50a923f0289efa2c2f"},"cell_type":"code","source":"engineered_test['first_active_month_timestamp'] = test['first_active_month'].apply(month_to_timestamp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c399a91f6027517a02daa623b2fcb870b5ef1b9e"},"cell_type":"code","source":"engineered_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0bfb461df0527a212d27a555cceba13d30f9ec74"},"cell_type":"code","source":"engineered_train_outlier.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2227a5df54ce846fc5955c4329278f884dcb709b"},"cell_type":"code","source":"engineered_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d0a3a4d2ca06398e63d8a2426001eb29523977b8"},"cell_type":"code","source":"plt.figure()\nengineered_train['first_active_month_timestamp'].value_counts().sort_index().plot(label='train')\nengineered_test['first_active_month_timestamp'].value_counts().sort_index().plot(label='test')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d495cac68c2a72cad357fdbee30c865ab59a3bc2"},"cell_type":"code","source":"plt.figure()\nengineered_train.groupby('first_active_month_timestamp')['target'].mean().plot()\nplt.title('mean target given timestamp')\nplt.ylabel('mean target')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e0f516106602903cbb51e1e1b031f6c1b1585bbe"},"cell_type":"code","source":"plt.figure()\nengineered_train.groupby('first_active_month_timestamp')['target'].var().plot()\nplt.title('var target given timestamp')\nplt.ylabel('var target')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2b6e25aa93d3221740550ad8d4e24b044bc69522"},"cell_type":"markdown","source":"It seems that time indicates both the mean and variance of the target value"},{"metadata":{"_uuid":"3f51c4903ee7cfe35ea8f8d96231ac3174032373"},"cell_type":"markdown","source":"# Now what does new_merchant_transactions give us?"},{"metadata":{"trusted":true,"_uuid":"ac7ef659d1a4a08e2f9c091f2d0997fee8e9b44d"},"cell_type":"code","source":"new_merchant_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f88a506994424861827ac5109b0861164a00147"},"cell_type":"code","source":"city_counts = new_merchant_transactions['city_id'].value_counts()\nunique_city = len(city_counts)\nmax_city = city_counts.idxmax()\nmax_count = city_counts[max_city]\nmax_percentage = max_count / len(new_merchant_transactions)\nprint('unique city: ', unique_city)\nprint('majority city: ', max_city)\nprint('count: ', max_count)\nprint('percentage: ', max_percentage)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a99f0f5b66b345b2e10d9d07a8c84500b31a5d8"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(15,5))\ntrain.merge(new_merchant_transactions[new_merchant_transactions['city_id'] == 69].groupby('card_id')['installments'].mean().to_frame(), how='inner', on='card_id')['target'].hist(bins=40, ax=ax[0])\ntrain.merge(new_merchant_transactions[new_merchant_transactions['city_id'] != 69].groupby('card_id')['installments'].mean().to_frame(), how='inner', on='card_id')['target'].hist(bins=40, ax=ax[1])\nax[0].set_title('with city_id as 69')\nax[1].set_title('with city_id as not 69')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1443a66600dd63d5a18107dc13ff7eef9684e72f"},"cell_type":"code","source":"plt.figure(figsize=(20,4))\nnew_merchant_transactions['subsector_id'].value_counts().sort_index().plot(kind='bar')\nplt.title('Distribution of subsector_id in new_merchant_transactions')\nplt.xlabel('subsector_id')\nplt.ylabel('count')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad903fa4db7fa8cd101bd4b67b2b92d370ead985"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(24,8))\ntrain.merge(new_merchant_transactions.groupby('card_id')['subsector_id'].nunique().to_frame(), how='inner', on='card_id').plot.scatter(y='target', x='subsector_id', ax=ax[0])\nengineered_train.merge(new_merchant_transactions.groupby('card_id')['subsector_id'].nunique().to_frame(), how='inner', on='card_id').plot.scatter(y='target', x='subsector_id', ax=ax[1])\nax[0].set_title('for all the training data')\nax[1].set_title('for training data without outliers')\nfig.suptitle('target and subsector_id coverage')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dd4ec3b294fe8a9e04c816807b225fbc0332f4e2"},"cell_type":"markdown","source":"Hmmmmm, but is this just because it is a indication of more purchase?"},{"metadata":{"trusted":true,"_uuid":"344d9c8651045d17353a4e823e88e358f22b1b3c"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(24,8))\ntrain.merge(new_merchant_transactions.groupby('card_id')['purchase_amount'].mean().to_frame(), how='inner', on='card_id').plot.scatter(y='target', x='purchase_amount', ax=ax[0])\nengineered_train.merge(new_merchant_transactions.groupby('card_id')['purchase_amount'].mean().to_frame(), how='inner', on='card_id').plot.scatter(y='target', x='purchase_amount', ax=ax[1])\nax[0].set_title('for all the training data')\nax[1].set_title('for training data without outliers')\nfig.suptitle('target and mean purchase amount')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b2b61773de1f8f8b41f2a4a7c9234ef98f6f562"},"cell_type":"code","source":"engineered_train['new_purchase_mean'] = engineered_train.merge(new_merchant_transactions.groupby('card_id')['purchase_amount'].mean().to_frame(), how='left', on='card_id')['purchase_amount']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef73164877b51fe9f89152427310370e97f6d914"},"cell_type":"code","source":"engineered_test['new_purchase_mean'] = engineered_test.merge(new_merchant_transactions.groupby('card_id')['purchase_amount'].mean().to_frame(), how='left', on='card_id')['purchase_amount']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"29aed774d8ad1946b2db494d1782bc3e9f7b7733"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(24,8))\ntrain.merge(new_merchant_transactions.groupby('card_id')['purchase_amount'].sum().to_frame(), how='inner', on='card_id').plot.scatter(y='target', x='purchase_amount', ax=ax[0])\nengineered_train.merge(new_merchant_transactions.groupby('card_id')['purchase_amount'].sum().to_frame(), how='inner', on='card_id').plot.scatter(y='target', x='purchase_amount', ax=ax[1])\nax[0].set_title('for all the training data')\nax[1].set_title('for training data without outliers')\nfig.suptitle('target and total purchase amount')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43cad9ddd913fffb2fa09e3f92df453748acf2ba"},"cell_type":"code","source":"engineered_train['new_purchase_sum'] = engineered_train.merge(new_merchant_transactions.groupby('card_id')['purchase_amount'].sum().to_frame(), how='left', on='card_id')['purchase_amount'].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d48d75b9f9e021ecdaa60a89964d8cdc6a462df"},"cell_type":"code","source":"engineered_test['new_purchase_sum'] = engineered_test.merge(new_merchant_transactions.groupby('card_id')['purchase_amount'].sum().to_frame(), how='left', on='card_id')['purchase_amount'].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb95faba5289e56ec72f088e451b580c6343c454"},"cell_type":"code","source":"plt.figure()\nnew_merchant_transactions['category_3'].value_counts().sort_index().plot(kind='bar')\nplt.title('Distribution of category_3 in new_merchant_transactions')\nplt.xlabel('category_3')\nplt.ylabel('count')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e94f3f300ade86ed623b041172318f6e645f844"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 3, figsize=(15,5))\ntrain.merge(new_merchant_transactions[new_merchant_transactions['category_3'] == 'A'].groupby('card_id')['installments'].mean().to_frame(), how='inner', on='card_id')['target'].hist(bins=20, ax=ax[0])\ntrain.merge(new_merchant_transactions[new_merchant_transactions['category_3'] == 'B'].groupby('card_id')['installments'].mean().to_frame(), how='inner', on='card_id')['target'].hist(bins=20, ax=ax[1])\ntrain.merge(new_merchant_transactions[new_merchant_transactions['category_3'] == 'C'].groupby('card_id')['installments'].mean().to_frame(), how='inner', on='card_id')['target'].hist(bins=20, ax=ax[2])\nax[0].set_title('with category_2 as A')\nax[1].set_title('with category_2 as B')\nax[2].set_title('with category_2 as C')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"040837305a57cd07da08a5a9dc5921c5d36de400"},"cell_type":"code","source":"plt.figure()\nnew_merchant_transactions['category_2'].value_counts().sort_index().plot(kind='bar')\nplt.title('Distribution of category_2 in new_merchant_transactions')\nplt.xlabel('category_2')\nplt.ylabel('count')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"faef5e79e53763945faf5add5b658c354c27e1ee"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 5, figsize=(15,3))\ntrain.merge(new_merchant_transactions[new_merchant_transactions['category_2'] == 1].groupby('card_id')['installments'].mean().to_frame(), how='inner', on='card_id')['target'].hist(bins=20, ax=ax[0])\ntrain.merge(new_merchant_transactions[new_merchant_transactions['category_2'] == 2].groupby('card_id')['installments'].mean().to_frame(), how='inner', on='card_id')['target'].hist(bins=20, ax=ax[1])\ntrain.merge(new_merchant_transactions[new_merchant_transactions['category_2'] == 3].groupby('card_id')['installments'].mean().to_frame(), how='inner', on='card_id')['target'].hist(bins=20, ax=ax[2])\ntrain.merge(new_merchant_transactions[new_merchant_transactions['category_2'] == 4].groupby('card_id')['installments'].mean().to_frame(), how='inner', on='card_id')['target'].hist(bins=20, ax=ax[3])\ntrain.merge(new_merchant_transactions[new_merchant_transactions['category_2'] == 5].groupby('card_id')['installments'].mean().to_frame(), how='inner', on='card_id')['target'].hist(bins=20, ax=ax[4])\nax[0].set_title('with category_2 as 1')\nax[1].set_title('with category_2 as 2')\nax[2].set_title('with category_2 as 3')\nax[3].set_title('with category_2 as 4')\nax[4].set_title('with category_2 as 5')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"23bf5b4ad71b248df99c60d6b9d37504f6ee3616"},"cell_type":"code","source":"plt.figure()\nnew_merchant_transactions['category_1'].value_counts().sort_index().plot(kind='bar')\nplt.title('Distribution of category_1 in new_merchant_transactions')\nplt.xlabel('category_1')\nplt.ylabel('count')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b5f015eabf285b35ccac006e1c0020546eced9b"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(10,4))\ntrain.merge(new_merchant_transactions[new_merchant_transactions['category_1'] == 'N'].groupby('card_id')['installments'].mean().to_frame(), how='inner', on='card_id')['target'].hist(bins=20, ax=ax[0])\ntrain.merge(new_merchant_transactions[new_merchant_transactions['category_1'] == 'Y'].groupby('card_id')['installments'].mean().to_frame(), how='inner', on='card_id')['target'].hist(bins=20, ax=ax[1])\nax[0].set_title('with category_1 as Y')\nax[1].set_title('with category_1 as N')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c932e27dc270cb5dca0f0e62fd27344543893277"},"cell_type":"code","source":"plt.figure()\nnew_merchant_transactions['installments'].value_counts().sort_index().plot(kind='bar')\nplt.title('Distribution of installments in new_merchant_transactions')\nplt.xlabel('installments')\nplt.ylabel('count')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f26ebdbaa8b6ac9d22af80e368520455cd48e2bc"},"cell_type":"markdown","source":"1. Huh? 999 and -1? What is that exactly?"},{"metadata":{"trusted":true,"_uuid":"589caa8c735cf857f02bddecd05117da86057968"},"cell_type":"code","source":"new_merchant_transactions[(new_merchant_transactions['installments'] == -1) | (new_merchant_transactions['installments'] == 999)]['installments'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75982e2c59a97ba8e6275a79437e985b4d21040e"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(10,4))\ntrain.merge(new_merchant_transactions[new_merchant_transactions['installments'] == -1].groupby('card_id')['installments'].mean().to_frame(), how='inner', on='card_id')['target'].hist(bins=20, ax=ax[0])\ntrain.merge(new_merchant_transactions[new_merchant_transactions['installments'] != -1].groupby('card_id')['installments'].mean().to_frame(), how='inner', on='card_id')['target'].hist(bins=20, ax=ax[1])\nax[0].set_title('with installments as -1')\nax[1].set_title('with installments as not -1')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f52ec63dabf7727a6f7703594c298e887e1e33ea"},"cell_type":"markdown","source":"So -1 might just mean it is missing. It doesn't indicate anything. For 999, there are too few in new_merchant_transactions. Will have to find out in historical one"},{"metadata":{"trusted":true,"_uuid":"5cfe33c4dae9083a8de001d2d6a4eb09fe070dc4"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(10,4))\ntrain.merge(new_merchant_transactions[(new_merchant_transactions['installments'] == 0) | (new_merchant_transactions['installments'] == 1)].groupby('card_id')['installments'].mean().to_frame(), how='inner', on='card_id')['target'].hist(bins=20, ax=ax[0])\ntrain.merge(new_merchant_transactions[(new_merchant_transactions['installments'] != 0) & (new_merchant_transactions['installments'] != 1)].groupby('card_id')['installments'].mean().to_frame(), how='inner', on='card_id')['target'].hist(bins=20, ax=ax[1])\nax[0].set_title('with installments as 0 or 1')\nax[1].set_title('with installments as otherwise')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cd62d3b88ac3a58264221352b18330d2203f8862"},"cell_type":"markdown","source":"Well... very likely it is another useless feature"},{"metadata":{"trusted":true,"_uuid":"9ba0e92b148c16ade3934f12ee12a6ddbd6ad9fa"},"cell_type":"code","source":"new_merchant_transactions['authorized_flag'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"22323b2b342d9708fc646de8d5a019daeb298de4"},"cell_type":"markdown","source":"Wat? really? Maybe only historial data make some differentiation in this field?"},{"metadata":{"trusted":true,"_uuid":"4991e98cffa63e2a3c53bf8241cb659233e75ca7"},"cell_type":"code","source":"plt.figure()\nnew_merchant_transactions['state_id'].value_counts().sort_index().plot(kind='bar')\nplt.title('Distribution of state_id in new_merchant_transactions')\nplt.xlabel('state_id')\nplt.ylabel('count')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b467f8b7e006f2c4189ae370c141aedff198442d"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(10,4))\ntrain.merge(new_merchant_transactions[new_merchant_transactions['state_id'] == 9].groupby('card_id')['installments'].mean().to_frame(), how='inner', on='card_id')['target'].hist(bins=20, ax=ax[0])\ntrain.merge(new_merchant_transactions[new_merchant_transactions['state_id'] != 9].groupby('card_id')['installments'].mean().to_frame(), how='inner', on='card_id')['target'].hist(bins=20, ax=ax[1])\nax[0].set_title('with state_id as 9')\nax[1].set_title('with state_id as not 9')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b17115bd244ed6f374b7c42e992c53cec02c21c"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(10,4))\nengineered_train.merge(new_merchant_transactions[new_merchant_transactions['state_id'] == 9].groupby('card_id')['installments'].mean().to_frame(), how='inner', on='card_id')['target'].hist(bins=40, ax=ax[0])\nengineered_train.merge(new_merchant_transactions[new_merchant_transactions['state_id'] != 9].groupby('card_id')['installments'].mean().to_frame(), how='inner', on='card_id')['target'].hist(bins=40, ax=ax[1])\nax[0].set_title('with state_id as 9')\nax[1].set_title('with state_id as not 9')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4da96dcb16103dcce995cf157ef21080291735d"},"cell_type":"code","source":"unique_in_new_merchant_transactions = len(new_merchant_transactions['card_id'].value_counts())\nunique_in_train = len(engineered_train['card_id'].value_counts())\ncommon_in_two = len(pd.merge(engineered_train, new_merchant_transactions.groupby('card_id')['month_lag'].mean().to_frame(), on='card_id', how='inner'))\nprint('unique card ids in new_merchant_transactions: ', unique_in_new_merchant_transactions)\nprint('unique card ids in train: ', unique_in_train)\nprint('common values: ', common_in_two)\nprint('missing values: ', unique_in_train - common_in_two)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"793f396c1f4a6fb55fc0bfd4011503ae91583fc8"},"cell_type":"code","source":"mean_month_lag = new_merchant_transactions['month_lag'].mean()\nprint('mean month_lag: ', mean_month_lag)\nengineered_train['month_lag'] = train[train['target'] > -20].merge(new_merchant_transactions.groupby('card_id')['month_lag'].mean().to_frame(), how='left', on='card_id')['month_lag'].fillna(mean_month_lag)\nengineered_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb2b35bf934893a18ddfad205c6a36a31f5aca34"},"cell_type":"code","source":"engineered_train_outlier['month_lag'] = train[train['target'] < -20].merge(new_merchant_transactions.groupby('card_id')['month_lag'].mean().to_frame(), how='left', on='card_id')['month_lag'].fillna(mean_month_lag)\nengineered_train_outlier.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d81203edeb3e39af07fa072fd0a819df612c38d"},"cell_type":"code","source":"plt.figure()\nengineered_train['month_lag'].hist(bins=40)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4460393e54a6ea32998ff4dd44f8ded3236bf25d"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(10,4))\nengineered_train[engineered_train['month_lag'] == 1.0]['target'].hist(bins=20, ax=ax[0])\nengineered_train[engineered_train['month_lag'] == 2.0]['target'].hist(bins=20, ax=ax[1])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"188a28f87e6e556b8959699e6d0749b67ca2fe9f"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 10, figsize=(30,3))\nfor i in range(10):\n    title = str(1.0+0.1*i) + '~' + str(1.0+0.1*(i+1))\n    engineered_train[(engineered_train['month_lag'] > 1.0+0.1*i) & (engineered_train['month_lag'] < 1.0+0.1*(i+1))]['target'].hist(bins=20, ax=ax[i])\n    ax[i].set_title(title)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1ab775fff974a85c80ed38d4058a958fdee7b898"},"cell_type":"markdown","source":"This tells that although it is mostly normal distribution, but the two extremes encodes some information"},{"metadata":{"_uuid":"f21898a46c1f1f9e1c5fe2fbb5f06ba78827211e"},"cell_type":"markdown","source":"# Now historical data. Please give me something useful :("},{"metadata":{"trusted":true,"_uuid":"99c1a48f69537e5f8a948b53cbc636b125109a52"},"cell_type":"code","source":"historical_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1029a4c7d7d77d8d4564d2c86c58ebb27ad9e950"},"cell_type":"code","source":"installments_999 = historical_transactions[historical_transactions['installments'] == 999]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2edb1a82aec1341abf1b5c7754c3fdacdac52c80"},"cell_type":"code","source":"plt.figure()\ninstallments_999['authorized_flag'].value_counts().sort_index().plot(kind='bar')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f047fbacd7cd633f0101e5d3d51423de49d0228"},"cell_type":"markdown","source":"Hmmm mystery solved. it seems that 999 is kind of spam that cannot get authorized, but is that useful?"},{"metadata":{"trusted":true,"_uuid":"ba7c464c67654b7377947d3578494ec141eec35c"},"cell_type":"code","source":"count_999 = len(historical_transactions[historical_transactions['installments'] == 999])\nunauthorized_count = len(historical_transactions[historical_transactions['authorized_flag'] == 'N'])\nprint('count_999: ', count_999)\nprint('unauthorized_count: ', unauthorized_count)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96a7573347e23b6d679d2d7da62dfcc380e7df54"},"cell_type":"code","source":"installments_999_unique_cards = installments_999.groupby('card_id')['installments'].mean().to_frame()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5e6849b239bb0a59c1c50766e44f31e6e82819c"},"cell_type":"code","source":"print('total unique 999 cards: ', len(installments_999_unique_cards))\ninstallments_999_unique_cards.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b42522aa0dfdf3ba7639707350183a183d70cab7"},"cell_type":"code","source":"plt.figure()\ntrain.merge(installments_999_unique_cards, how='inner', on='card_id')['target'].hist(bins=40)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c587e3803ba6932f2291b4478a2922fd55aca3f7"},"cell_type":"markdown","source":"hmmmmm what can I say ..."},{"metadata":{"trusted":true,"_uuid":"041ef1cf1727b272324f7047492027111192973d"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(24,8))\ntrain.merge(historical_transactions.groupby('card_id')['purchase_amount'].mean().to_frame(), how='inner', on='card_id').plot.scatter(y='target', x='purchase_amount', ax=ax[0])\nengineered_train.merge(historical_transactions.groupby('card_id')['purchase_amount'].mean().to_frame(), how='inner', on='card_id').plot.scatter(y='target', x='purchase_amount', ax=ax[1])\nax[0].set_title('for all the training data')\nax[0].set_xlim([-10, 90])\nax[1].set_title('for training data without outliers')\nax[1].set_xlim([-10, 90])\nfig.suptitle('target and mean purchase amount')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8dbe1e2ef7975572f970e692ace69502cdea243f"},"cell_type":"code","source":"engineered_train['history_purchase_mean'] = engineered_train.merge(historical_transactions.groupby('card_id')['purchase_amount'].mean().to_frame(), how='left', on='card_id')['purchase_amount']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54bc0abcb5747b70af9451751844c7c680cbe7df"},"cell_type":"code","source":"engineered_test['history_purchase_mean'] = engineered_test.merge(historical_transactions.groupby('card_id')['purchase_amount'].mean().to_frame(), how='left', on='card_id')['purchase_amount']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77a503df56cdc5b14d440e182e7525163872d6f3"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(24,8))\ntrain.merge(historical_transactions.groupby('card_id')['purchase_amount'].sum().to_frame(), how='inner', on='card_id').plot.scatter(y='target', x='purchase_amount', ax=ax[0])\nengineered_train.merge(historical_transactions.groupby('card_id')['purchase_amount'].sum().to_frame(), how='inner', on='card_id').plot.scatter(y='target', x='purchase_amount', ax=ax[1])\nax[0].set_title('for all the training data')\nax[0].set_xlim([-800, 500])\nax[1].set_title('for training data without outliers')\nax[1].set_xlim([-800, 500])\nfig.suptitle('target and mean purchase amount')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d636a8dac814cdfee2090a063184452e62fad3c8"},"cell_type":"code","source":"engineered_train['history_purchase_sum'] = engineered_train.merge(historical_transactions.groupby('card_id')['purchase_amount'].sum().to_frame(), how='left', on='card_id')['purchase_amount'].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eea3b844a939199bc27536f8c8cfdb06de3a1dc2"},"cell_type":"code","source":"engineered_test['history_purchase_sum'] = engineered_test.merge(historical_transactions.groupby('card_id')['purchase_amount'].sum().to_frame(), how='left', on='card_id')['purchase_amount'].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f727483e722c21af9c295d9eae2e11e1aa5b25d6"},"cell_type":"markdown","source":"# Do some preprocessing to make life easier"},{"metadata":{"trusted":true,"_uuid":"1a78bd8f8855eae92569d69c7249d2a10a334b04"},"cell_type":"code","source":"min_first_active_month = min(engineered_train['first_active_month_timestamp'].min(), engineered_test['first_active_month_timestamp'].min())\nmax_first_active_month = max(engineered_train['first_active_month_timestamp'].max(), engineered_test['first_active_month_timestamp'].max())\nprint('first_active_month range: ', min_first_active_month, ' ~ ', max_first_active_month)\nfirst_active_month_range = max_first_active_month - min_first_active_month\nprint('range: ', first_active_month_range)\nplt.figure()\nengineered_test['first_active_month_timestamp'].hist()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ba7b977fed24d59ca7fe1d3020bd6c9abf77d94"},"cell_type":"code","source":"def normalize_timestamp(timestamp):\n    return (timestamp - min_first_active_month) / first_active_month_range","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"012bb475c1b6d4e728f49e8a60e2ee2d05ef63ba"},"cell_type":"code","source":"engineered_train['normalized_timestamp'] = engineered_train['first_active_month_timestamp'].apply(normalize_timestamp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6460da8a2bcdfe4c9c53950e1f7af0a280c9751d"},"cell_type":"code","source":"engineered_test['normalized_timestamp'] = engineered_test['first_active_month_timestamp'].apply(normalize_timestamp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"21f9074e279f8166ce09b73d015b31df03887996"},"cell_type":"code","source":"def normalize_target(target):\n    return (target + 20) / 40","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9dd5ef289da4817289d5c637f33180ddd28a431"},"cell_type":"code","source":"engineered_train['normalized_target'] = engineered_train['target'].fillna(0).apply(normalize_target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e7962767157db88e0c6df3f6af1586ae832a23c"},"cell_type":"code","source":"plt.figure()\nengineered_train['normalized_target'].hist(bins=40)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5a58470ce3c58736f9eab897300908c5168a3fcd"},"cell_type":"code","source":"min_history_purchase_sum = min(engineered_train['history_purchase_sum'].min(), engineered_test['history_purchase_sum'].min())\nmax_history_purchase_sum = max(engineered_train['history_purchase_sum'].max(), engineered_test['history_purchase_sum'].max())\nhistory_purchase_sum_na = engineered_train['history_purchase_sum'].isna().count()\nhistory_purchase_sum_not_na = len(engineered_train) - history_purchase_sum_na\nprint('history_purchase_sum_na: ', history_purchase_sum_na)\nprint('history_purchase_sum_not_na: ', history_purchase_sum_not_na)\nprint('history_purchase_sum range: ', min_history_purchase_sum, ' ~ ', max_history_purchase_sum)\nhistory_purchase_sum_range = max_history_purchase_sum - min_history_purchase_sum\nprint('range: ', history_purchase_sum_range)\nplt.figure()\nengineered_train['history_purchase_sum'].hist()\nplt.show()\nplt.figure()\nengineered_train[(engineered_train['history_purchase_sum'].fillna(0) < 200) & (engineered_train['history_purchase_sum'].fillna(0) > -500)]['history_purchase_sum'].hist(bins=40)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8955d5df96aa94fbd102436bec558c93f3b9fa15"},"cell_type":"code","source":"def normalize_history_purchase_sum(history_purchase_sum):\n    if history_purchase_sum < -500.0:\n        return 0\n    elif history_purchase_sum > 200.0:\n        return 1\n    else:\n        return (history_purchase_sum + 500.0) / 700.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"08b08dd227fcc615fada849db1a154c63649f329"},"cell_type":"code","source":"engineered_train['normalized_history_purchase_sum'] = engineered_train['history_purchase_sum'].fillna(0).apply(normalize_history_purchase_sum)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e7c2b69082d435fe92489644c1f324f3b9fea0e8"},"cell_type":"code","source":"engineered_test['normalized_history_purchase_sum'] = engineered_test['history_purchase_sum'].fillna(0).apply(normalize_history_purchase_sum)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d7caac52fc048b53c6500847cbe9d1ebb73b1bd"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(16,5))\nengineered_train['normalized_history_purchase_sum'].hist(ax=ax[0], bins=40)\nengineered_test['normalized_history_purchase_sum'].hist(ax=ax[1], bins=40)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc6d9469fe5aeef21829aec29a920a78f108d8d3"},"cell_type":"code","source":"min_history_purchase_mean = min(engineered_train['history_purchase_mean'].min(), engineered_test['history_purchase_mean'].min())\nmax_history_purchase_mean = max(engineered_train['history_purchase_mean'].max(), engineered_test['history_purchase_mean'].max())\nprint('max_history_purchase_mean range: ', min_history_purchase_mean, ' ~ ', max_history_purchase_mean)\nhistory_purchase_mean_range = max_history_purchase_mean - min_history_purchase_mean\nprint('range: ', history_purchase_mean_range)\nplt.figure()\nengineered_train[(engineered_train['history_purchase_mean'].fillna(0) < 2) & (engineered_train['history_purchase_mean'].fillna(0) > -1)]['history_purchase_mean'].hist(bins=40)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f56fea130e129d104e0a720e975060d2d8c62c8"},"cell_type":"code","source":"def normalize_history_purchase_mean(history_purchase_mean):\n    if history_purchase_mean < -1:\n        return 0\n    elif history_purchase_mean > 2:\n        return 1\n    return (history_purchase_mean + 1) / 3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59ebc1b2cac6c0a7e012f4d3270255816d71b68a"},"cell_type":"code","source":"engineered_train['normalized_history_purchase_mean'] = engineered_train['history_purchase_mean'].fillna(0).apply(normalize_history_purchase_mean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a713a00a6c486ea7a36783d95bedd14c1352fc7"},"cell_type":"code","source":"engineered_test['normalized_history_purchase_mean'] = engineered_test['history_purchase_mean'].fillna(0).apply(normalize_history_purchase_mean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d10a1e833e21c8f9d1d17919c0e044207f7163bc"},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(16,5))\nengineered_train['normalized_history_purchase_mean'].hist(ax=ax[0], bins=40)\nengineered_test['normalized_history_purchase_mean'].hist(ax=ax[1], bins=40)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f7113e26fed7d253e689b72b00d11a4d4506fb26"},"cell_type":"markdown","source":"# This section outputs some files that can be used instead of the original files"},{"metadata":{"trusted":true,"_uuid":"849eb2c9a00a8957672c94e5912fdb7345b250d2"},"cell_type":"code","source":"historical_transactions[:100].to_csv('light_historical_transactions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3170fed12854df674310f9e4c07e12c9151bf50"},"cell_type":"code","source":"engineered_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d0929064017a8d1636e2091bd983f3aa07980f79"},"cell_type":"code","source":"engineered_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cfa4754eaa7d6f221a824c6d628e85a526afbab5"},"cell_type":"code","source":"engineered_train[:100].to_csv('light_engineered_train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"985ff5e168f0f7f27fc48f4a9b56472916278461"},"cell_type":"code","source":"engineered_test[:100].to_csv('light_engineered_test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"296a2a37486f456152389d0c62e4a5c4bf215970"},"cell_type":"code","source":"engineered_train.to_csv('engineered_train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e2937fb6426a1fa1a5e456408da65cc9aac721d"},"cell_type":"code","source":"engineered_test.to_csv('engineered_test.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"981a52784f7f169aeb0e4a2e1434bf904c82b1d1"},"cell_type":"markdown","source":"# Do some sanity check to see if it make sense at all"},{"metadata":{"trusted":true,"_uuid":"626a9ff7fb3e8013f43b22a1ccbbdf1484ef4a74"},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense, Activation\nfrom keras.layers.normalization import BatchNormalization","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"52a553acfcd79b1803cf0d119d0392083a7c583b"},"cell_type":"code","source":"model = Sequential()\nmodel.add(Dense(64, activation='relu', input_dim=3))\nmodel.add(Dense(32, activation='relu'))\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.compile(optimizer='sgd', loss='mean_squared_error')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95bbb14d179d8c61c47a619e7737bbfff058a4a8"},"cell_type":"code","source":"tags = ['normalized_timestamp', 'normalized_history_purchase_sum', 'normalized_history_purchase_mean']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96171e1ada3a5b95073facac66ce519a7d9044f7"},"cell_type":"code","source":"training_set = engineered_train[tags].values\ntraining_set","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59becfe9fa0e9dfcfe7efd252aa4a1da4b0da169"},"cell_type":"code","source":"training_target = engineered_train['target'].values\ntraining_target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a8ad57825b9f506bef2d221981c91a002f33a58a"},"cell_type":"code","source":"testing_set = engineered_test[tags].values\ntesting_set","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9de76b694cf49a6b038a8b3428ca198ea0554d0"},"cell_type":"code","source":"history = model.fit(x=training_set, y=training_target, epochs=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9e633995c81bb1cf85b373836e0c8dd2f53d034"},"cell_type":"code","source":"pred = model.predict(x=testing_set, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b367198664c99435eceffa0559a07a9b4c28af34"},"cell_type":"code","source":"pred.flatten()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"86c116428fdc503ecc0536f874df5b76e2db0d1f"},"cell_type":"code","source":"print('prediction length: ', len(pred.flatten().tolist()))\nprint('engineered test length: ', len(engineered_test['card_id']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07d755f3b73afd258e0a3dda57fbf8eadb85f2ee"},"cell_type":"code","source":"output_result = pd.DataFrame({'card_id': engineered_test['card_id'].values.tolist(), 'target': pred.flatten().tolist()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6258833879daff558f5bee7b56ce2acb12fbd052"},"cell_type":"code","source":"output_result.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"29f565d11abefe55a84a4a549a75eb0fc2a51c70"},"cell_type":"code","source":"def denormalize_target(target):\n    return target * 40 - 20","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e54fa708555c9ac05b52d469649c219458f86d6a"},"cell_type":"code","source":"output_result['target'] = engineered_train['target'].fillna(0.5).apply(normalize_target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"acd38126ef4c0fcc4ad98a69611bb8066ec6c9fd"},"cell_type":"code","source":"output_result.fillna(0.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a548d1ee611a17ed08b8dc74aaf55e4170792684"},"cell_type":"code","source":"output_result.fillna(0.0).to_csv('output_result.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"344f545ad51e1cade2198b53db9f2a520b242091"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}