{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n        \n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-25T01:13:37.864576Z","iopub.execute_input":"2022-02-25T01:13:37.866406Z","iopub.status.idle":"2022-02-25T01:13:37.899998Z","shell.execute_reply.started":"2022-02-25T01:13:37.866263Z","shell.execute_reply":"2022-02-25T01:13:37.898981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nprint(\"Tensorflow version \" + tf.__version__)\n\ntry:\n  tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\n  print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])\nexcept ValueError:\n  raise BaseException('ERROR: Not connected to a TPU runtime; please see the previous cell in this notebook for instructions!')\n\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\ntpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:41:23.849435Z","iopub.execute_input":"2022-02-25T00:41:23.850225Z","iopub.status.idle":"2022-02-25T00:41:35.051575Z","shell.execute_reply.started":"2022-02-25T00:41:23.850170Z","shell.execute_reply":"2022-02-25T00:41:35.050956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\ndevice_name = tf.test.gpu_device_name()\nif device_name != '/device:GPU:0':\n  raise SystemError('GPU device not found')\nprint('Found GPU at: {}'.format(device_name))","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:23:45.162265Z","iopub.execute_input":"2022-02-25T00:23:45.162501Z","iopub.status.idle":"2022-02-25T00:23:45.295098Z","shell.execute_reply.started":"2022-02-25T00:23:45.162474Z","shell.execute_reply":"2022-02-25T00:23:45.286586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()\n","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:23:45.296104Z","iopub.status.idle":"2022-02-25T00:23:45.296488Z","shell.execute_reply.started":"2022-02-25T00:23:45.296307Z","shell.execute_reply":"2022-02-25T00:23:45.296327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport timeit\n\ndevice_name = tf.test.gpu_device_name()\nif device_name != '/device:GPU:0':\n  print(\n      '\\n\\nThis error most likely means that this notebook is not '\n      'configured to use a GPU.  Change this in Notebook Settings via the '\n      'command palette (cmd/ctrl-shift-P) or the Edit menu.\\n\\n')\n  raise SystemError('GPU device not found')\n\ndef cpu():\n  with tf.device('/cpu:0'):\n    random_image_cpu = tf.random.normal((100, 100, 100, 3))\n    net_cpu = tf.keras.layers.Conv2D(32, 7)(random_image_cpu)\n    return tf.math.reduce_sum(net_cpu)\n\ndef gpu():\n  with tf.device('/device:GPU:0'):\n    random_image_gpu = tf.random.normal((100, 100, 100, 3))\n    net_gpu = tf.keras.layers.Conv2D(32, 7)(random_image_gpu)\n    return tf.math.reduce_sum(net_gpu)\n  \n# We run each op once to warm up; see: https://stackoverflow.com/a/45067900\ncpu()\ngpu()\n\n# Run the op several times.\nprint('Time (s) to convolve 32x7x7x3 filter over random 100x100x100x3 images '\n      '(batch x height x width x channel). Sum of ten runs.')\nprint('CPU (s):')\ncpu_time = timeit.timeit('cpu()', number=10, setup=\"from __main__ import cpu\")\nprint(cpu_time)\nprint('GPU (s):')\ngpu_time = timeit.timeit('gpu()', number=10, setup=\"from __main__ import gpu\")\nprint(gpu_time)\nprint('GPU speedup over CPU: {}x'.format(int(cpu_time/gpu_time)))","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:23:45.298113Z","iopub.status.idle":"2022-02-25T00:23:45.298465Z","shell.execute_reply.started":"2022-02-25T00:23:45.298300Z","shell.execute_reply":"2022-02-25T00:23:45.298317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:13:51.348328Z","iopub.execute_input":"2022-02-25T01:13:51.349218Z","iopub.status.idle":"2022-02-25T01:13:51.364426Z","shell.execute_reply.started":"2022-02-25T01:13:51.349161Z","shell.execute_reply":"2022-02-25T01:13:51.363506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport math as math\nimport datetime as dt\nimport sklearn\nimport plotly\nimport plotly.figure_factory as ff\nimport plotly.graph_objs as go\nfrom plotly import __version__\nprint(__version__)\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\ninit_notebook_mode(connected=True)\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\npd.set_option('precision', 5)\npd.options.display.float_format = '{:20,.2f}'.format\nnp.set_printoptions(suppress =True) \n\nimport os\nprint(os.listdir(\"../input\"))\nimport tqdm\nfrom tqdm import tqdm\n","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:14:04.562730Z","iopub.execute_input":"2022-02-25T01:14:04.562967Z","iopub.status.idle":"2022-02-25T01:14:04.575982Z","shell.execute_reply.started":"2022-02-25T01:14:04.562938Z","shell.execute_reply":"2022-02-25T01:14:04.575289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('../input/elo-merchant-category-recommendation/historical_transactions.csv', nrows=50).head(5)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:24:48.689071Z","iopub.execute_input":"2022-02-25T00:24:48.689501Z","iopub.status.idle":"2022-02-25T00:24:48.735588Z","shell.execute_reply.started":"2022-02-25T00:24:48.689465Z","shell.execute_reply":"2022-02-25T00:24:48.734718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = pd.read_csv('../input/elo-merchant-category-recommendation/historical_transactions.csv', nrows=50).head(5)\nx.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:24:48.737696Z","iopub.execute_input":"2022-02-25T00:24:48.738506Z","iopub.status.idle":"2022-02-25T00:24:48.761121Z","shell.execute_reply.started":"2022-02-25T00:24:48.738463Z","shell.execute_reply":"2022-02-25T00:24:48.760121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('../input/elo-merchant-category-recommendation/merchants.csv', nrows=50).head(3)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:24:48.762522Z","iopub.execute_input":"2022-02-25T00:24:48.762758Z","iopub.status.idle":"2022-02-25T00:24:48.793913Z","shell.execute_reply.started":"2022-02-25T00:24:48.762731Z","shell.execute_reply":"2022-02-25T00:24:48.792987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('../input/elo-merchant-category-recommendation/new_merchant_transactions.csv', nrows=50).head(3)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:24:48.795210Z","iopub.execute_input":"2022-02-25T00:24:48.795855Z","iopub.status.idle":"2022-02-25T00:24:48.824791Z","shell.execute_reply.started":"2022-02-25T00:24:48.795809Z","shell.execute_reply":"2022-02-25T00:24:48.824142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('../input/elo-merchant-category-recommendation/train.csv', nrows=50).head(3)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:24:48.826556Z","iopub.execute_input":"2022-02-25T00:24:48.826876Z","iopub.status.idle":"2022-02-25T00:24:48.852160Z","shell.execute_reply.started":"2022-02-25T00:24:48.826851Z","shell.execute_reply":"2022-02-25T00:24:48.851284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('../input/elo-merchant-category-recommendation/test.csv', nrows=50).head(3)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:24:48.853284Z","iopub.execute_input":"2022-02-25T00:24:48.853582Z","iopub.status.idle":"2022-02-25T00:24:48.875329Z","shell.execute_reply.started":"2022-02-25T00:24:48.853554Z","shell.execute_reply":"2022-02-25T00:24:48.874534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('../input/elo-merchant-category-recommendation/sample_submission.csv', nrows=50).head(3)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:24:48.876412Z","iopub.execute_input":"2022-02-25T00:24:48.876774Z","iopub.status.idle":"2022-02-25T00:24:48.897092Z","shell.execute_reply.started":"2022-02-25T00:24:48.876746Z","shell.execute_reply":"2022-02-25T00:24:48.896267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install openpyxl","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:24:48.898365Z","iopub.execute_input":"2022-02-25T00:24:48.898578Z","iopub.status.idle":"2022-02-25T00:24:59.030522Z","shell.execute_reply.started":"2022-02-25T00:24:48.898555Z","shell.execute_reply":"2022-02-25T00:24:59.029769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_excel('../input/elo-merchant-category-recommendation/Data_Dictionary.xlsx')","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:24:59.031684Z","iopub.execute_input":"2022-02-25T00:24:59.031944Z","iopub.status.idle":"2022-02-25T00:24:59.285625Z","shell.execute_reply.started":"2022-02-25T00:24:59.031914Z","shell.execute_reply":"2022-02-25T00:24:59.284751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reading the Data","metadata":{}},{"cell_type":"code","source":"df_train = reduce_mem_usage(pd.read_csv(\"../input/elo-merchant-category-recommendation/train.csv\"))\ndf_test = reduce_mem_usage(pd.read_csv(\"../input/elo-merchant-category-recommendation/test.csv\"))\ndf_historical =reduce_mem_usage(pd.read_csv(\"../input/elo-merchant-category-recommendation/historical_transactions.csv\",parse_dates=['purchase_date']))\ndf_new =reduce_mem_usage(pd.read_csv(\"../input/elo-merchant-category-recommendation/new_merchant_transactions.csv\",parse_dates=['purchase_date']))","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:42:30.192114Z","iopub.execute_input":"2022-02-25T00:42:30.192931Z","iopub.status.idle":"2022-02-25T00:44:22.872617Z","shell.execute_reply.started":"2022-02-25T00:42:30.192828Z","shell.execute_reply":"2022-02-25T00:44:22.871779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_historical.head(2)  #historical transactions","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:26:53.355311Z","iopub.execute_input":"2022-02-25T00:26:53.355645Z","iopub.status.idle":"2022-02-25T00:26:53.372774Z","shell.execute_reply.started":"2022-02-25T00:26:53.355605Z","shell.execute_reply":"2022-02-25T00:26:53.371963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_historical.card_id.unique())  # number of unique card IDs","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:26:53.374194Z","iopub.execute_input":"2022-02-25T00:26:53.374418Z","iopub.status.idle":"2022-02-25T00:26:55.938811Z","shell.execute_reply.started":"2022-02-25T00:26:53.374391Z","shell.execute_reply":"2022-02-25T00:26:55.937917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_new.head(2)   # new merchant transactions","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:26:55.939969Z","iopub.execute_input":"2022-02-25T00:26:55.940218Z","iopub.status.idle":"2022-02-25T00:26:55.956378Z","shell.execute_reply.started":"2022-02-25T00:26:55.940191Z","shell.execute_reply":"2022-02-25T00:26:55.955464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_historical=df_historical.loc[df_historical.authorized_flag==\"Y\",]\ndf_historical.purchase_amount += 0.75\ndf_new.purchase_amount += 0.75","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:26:55.961655Z","iopub.execute_input":"2022-02-25T00:26:55.961898Z","iopub.status.idle":"2022-02-25T00:27:02.191119Z","shell.execute_reply.started":"2022-02-25T00:26:55.961870Z","shell.execute_reply":"2022-02-25T00:27:02.190389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Checking for Purchase, Return and Rebate Transactions (*if any)","metadata":{}},{"cell_type":"markdown","source":"### As the purchase_amount variable is normalized it's not possible to check if the transaction is a Purchase/Return/Rebate one","metadata":{}},{"cell_type":"markdown","source":"### Here we are doing aggregation by card-id (At Customer Level). We Can also compute at Item Level RFM and Store Level RFM or even website visit & activity based RFM etc depending on the availability of data","metadata":{}},{"cell_type":"code","source":"def groupby_mean(x):                    # defining average transactions\n    return x.mean()\n\ndef groupby_count(x):                  # defining number of transactions\n    return x.count()\n\ndef purchase_duration(x):               # defining function purchase_durations as Duration of the purchase or time gap (in days between 2 transactions) \n    return (x.max() - x.min()).days\n\ndef avg_frequency(x):\n    return (x.max() - x.min()).days/x.count() # defining average frequency of transactions as purchase duration / count\n\ngroupby_mean.__name__ = 'avg'\ngroupby_count.__name__ = 'count'\npurchase_duration.__name__ = 'purchase_duration'\navg_frequency.__name__ = 'purchase_frequency'\n\ndef get_max(cols):\n    return max(cols[0],cols[1])","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:27:02.192431Z","iopub.execute_input":"2022-02-25T00:27:02.192654Z","iopub.status.idle":"2022-02-25T00:27:02.200736Z","shell.execute_reply.started":"2022-02-25T00:27:02.192629Z","shell.execute_reply":"2022-02-25T00:27:02.198855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_agg_Monetary = df_historical.groupby('card_id').agg({'purchase_amount':sum})   \ndf_agg_Monetary.columns = ['Monetary']\nprint(df_agg_Monetary.shape)\ndf_agg_Monetary.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:27:02.201944Z","iopub.execute_input":"2022-02-25T00:27:02.202276Z","iopub.status.idle":"2022-02-25T00:27:05.986724Z","shell.execute_reply.started":"2022-02-25T00:27:02.202248Z","shell.execute_reply":"2022-02-25T00:27:05.985911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_agg_Frequency = df_historical.groupby('card_id').agg({'card_id': groupby_count,'purchase_date': groupby_count})\ndf_agg_Frequency['Frequency'] = df_agg_Frequency[['card_id','purchase_date']].apply(get_max,axis = 1)\nprint(df_agg_Frequency.shape)\ndf_agg_Frequency.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:27:05.988327Z","iopub.execute_input":"2022-02-25T00:27:05.989191Z","iopub.status.idle":"2022-02-25T00:27:46.906326Z","shell.execute_reply.started":"2022-02-25T00:27:05.989124Z","shell.execute_reply":"2022-02-25T00:27:46.905474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_date = max(df_historical['purchase_date'])\nprint(max_date)\n\n# x = df_historical['purchase_date'][0]\n# (max_date-x).days","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:27:46.907686Z","iopub.execute_input":"2022-02-25T00:27:46.907922Z","iopub.status.idle":"2022-02-25T00:29:05.211070Z","shell.execute_reply.started":"2022-02-25T00:27:46.907895Z","shell.execute_reply":"2022-02-25T00:29:05.210208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_agg_Recency = df_historical.groupby(['card_id']).agg({'purchase_date':max})\ndf_agg_Recency['Recency'] = df_agg_Recency['purchase_date'].apply(lambda x:(max_date-x).days)\nprint(df_agg_Recency.shape)\ndf_agg_Recency.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:29:05.212686Z","iopub.execute_input":"2022-02-25T00:29:05.213688Z","iopub.status.idle":"2022-02-25T00:29:18.639078Z","shell.execute_reply.started":"2022-02-25T00:29:05.213645Z","shell.execute_reply":"2022-02-25T00:29:18.638119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_RFM = pd.merge(pd.merge(df_agg_Recency, df_agg_Frequency, left_index=True, right_index=True), \n                           df_agg_Monetary, left_index=True, right_index=True)\ndf_RFM.drop(columns=['purchase_date_x', 'card_id', 'purchase_date_y'], inplace=True)\ndf_RFM.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:29:18.640563Z","iopub.execute_input":"2022-02-25T00:29:18.641005Z","iopub.status.idle":"2022-02-25T00:29:19.252400Z","shell.execute_reply.started":"2022-02-25T00:29:18.640976Z","shell.execute_reply":"2022-02-25T00:29:19.251565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,8))\ncounts, bin_edges = np.histogram(np.log1p(df_RFM['Recency']), bins=10, density = True)\npdf = counts/(sum(counts))\nprint(pdf);\nprint(bin_edges);\ncdf = np.cumsum(pdf)\nplt.plot(bin_edges[1:],pdf);\nplt.plot(bin_edges[1:], cdf)\nplt.xlabel('log_Recency', fontsize=13)\nplt.ylabel('Percentage', fontsize=13)\nplt.title('Recency Cumulative Frequency Distribution',fontsize=15)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:29:19.253671Z","iopub.execute_input":"2022-02-25T00:29:19.253981Z","iopub.status.idle":"2022-02-25T00:29:19.581662Z","shell.execute_reply.started":"2022-02-25T00:29:19.253944Z","shell.execute_reply":"2022-02-25T00:29:19.580650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,8))\ncounts, bin_edges = np.histogram(np.log1p(df_RFM['Frequency']), bins=10, density = True)\npdf = counts/(sum(counts))\nprint(pdf);\nprint(bin_edges);\ncdf = np.cumsum(pdf)\nplt.plot(bin_edges[1:],pdf);\nplt.plot(bin_edges[1:], cdf)\nplt.xlabel('log_Frequency', fontsize=12)\nplt.ylabel('Percentage', fontsize=12)\nplt.title('Frequency Cumulative Frequency Distribution',fontsize=15)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:29:19.582993Z","iopub.execute_input":"2022-02-25T00:29:19.583361Z","iopub.status.idle":"2022-02-25T00:29:19.869033Z","shell.execute_reply.started":"2022-02-25T00:29:19.583313Z","shell.execute_reply":"2022-02-25T00:29:19.867976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,8))\ncounts, bin_edges = np.histogram(np.log1p(df_RFM['Monetary']), bins=10, density = True)\npdf = counts/(sum(counts))\nprint(pdf);\nprint(bin_edges);\ncdf = np.cumsum(pdf)\nplt.plot(bin_edges[1:],pdf);\nplt.plot(bin_edges[1:], cdf)\nplt.xlabel('log_Monetary', fontsize=12)\nplt.ylabel('Percentage', fontsize=12)\nplt.title('Monetary Cumulative Frequency Distribution',fontsize=15)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:29:19.870633Z","iopub.execute_input":"2022-02-25T00:29:19.871202Z","iopub.status.idle":"2022-02-25T00:29:20.157107Z","shell.execute_reply.started":"2022-02-25T00:29:19.871153Z","shell.execute_reply":"2022-02-25T00:29:20.156227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.close();\nsns.set_style(\"whitegrid\");\nsns.pairplot(df_RFM, size=4);\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:29:20.158238Z","iopub.execute_input":"2022-02-25T00:29:20.158467Z","iopub.status.idle":"2022-02-25T00:30:23.839515Z","shell.execute_reply.started":"2022-02-25T00:29:20.158442Z","shell.execute_reply":"2022-02-25T00:30:23.838809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Simple Rule Based Approach - The Business Analyst Way","metadata":{}},{"cell_type":"code","source":"df_RFM.quantile(q=[0.1,0.25,0.4,0.5,0.75,0.9])","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:30:23.840719Z","iopub.execute_input":"2022-02-25T00:30:23.841048Z","iopub.status.idle":"2022-02-25T00:30:23.881624Z","shell.execute_reply.started":"2022-02-25T00:30:23.841022Z","shell.execute_reply":"2022-02-25T00:30:23.880726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Thresholds for R | F | M","metadata":{}},{"cell_type":"code","source":"threshold_M = int(df_RFM.median()['Monetary'])+1\nprint(threshold_M)\nthreshold_F = df_RFM.Frequency.quantile(0.75)\nprint(threshold_F)\nthreshold_R = df_RFM.Recency.quantile(0.40)\nprint(threshold_R)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:30:23.882764Z","iopub.execute_input":"2022-02-25T00:30:23.882997Z","iopub.status.idle":"2022-02-25T00:30:23.915117Z","shell.execute_reply.started":"2022-02-25T00:30:23.882971Z","shell.execute_reply":"2022-02-25T00:30:23.914515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_RFM['threshold_R'] = df_RFM['Recency'].apply(lambda x: x < threshold_R)\ndf_RFM['threshold_F'] = df_RFM['Frequency'].apply(lambda x: x > threshold_F)\ndf_RFM['threshold_M'] = df_RFM['Monetary'].apply(lambda x: x > threshold_M)\ndf_RFM[['threshold_R', 'threshold_F', 'threshold_M']] = df_RFM[['threshold_R', 'threshold_F', 'threshold_M']].apply(lambda x: x.astype(int), axis=1)\ndf_RFM['IsLoyal'] = 'NA'\ndf_RFM['Segment'] = 'NA'\ndf_RFM.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:30:23.916366Z","iopub.execute_input":"2022-02-25T00:30:23.916771Z","iopub.status.idle":"2022-02-25T00:31:15.520675Z","shell.execute_reply.started":"2022-02-25T00:30:23.916742Z","shell.execute_reply":"2022-02-25T00:31:15.519767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Loyalty_assign(x):\n    if((x[5]==1) & (x[4]==1)):\n        return 'Loyal'\n       \n    elif((x[3]==1) & (x[4]==0)):\n        return 'Loyal'\n         \n    else:\n        return 'Not Loyal'\n    \ndef Segment_assign(x):\n    if((x[5]==1) & (x[3]==1) & (x[4]==1)):\n        return 'Champions'\n       \n    elif((x[5]==1) & (x[3]==1) & (x[4]==0)):\n        return 'Future Champions'\n         \n    elif((x[5]==1) & (x[3]==0) & (x[4]==1)):\n        return 'Very Valuable'\n         \n    elif((x[5]==1) & (x[3]==0) & (x[4]==0)):\n        return 'Hibernating'\n         \n    elif((x[5]==0) & (x[3]==1) & (x[4]==1)):\n        return 'Active'\n         \n    elif((x[5]==0) & (x[3]==1) & (x[4]==0)):\n        return 'About to Sleep'\n         \n    elif((x[5]==0) & (x[3]==0)):\n        return 'Lost'","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:31:15.521932Z","iopub.execute_input":"2022-02-25T00:31:15.522188Z","iopub.status.idle":"2022-02-25T00:31:15.533588Z","shell.execute_reply.started":"2022-02-25T00:31:15.522158Z","shell.execute_reply":"2022-02-25T00:31:15.532613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_RFM['Segment'] = df_RFM.apply(Segment_assign, axis=1)\ndf_RFM['IsLoyal'] = df_RFM.apply(Loyalty_assign, axis=1)\ndf_RFM.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:31:15.534927Z","iopub.execute_input":"2022-02-25T00:31:15.535335Z","iopub.status.idle":"2022-02-25T00:31:37.575095Z","shell.execute_reply.started":"2022-02-25T00:31:15.535296Z","shell.execute_reply":"2022-02-25T00:31:37.574262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_RFM['IsLoyal'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:31:37.576471Z","iopub.execute_input":"2022-02-25T00:31:37.577189Z","iopub.status.idle":"2022-02-25T00:31:37.622916Z","shell.execute_reply.started":"2022-02-25T00:31:37.577155Z","shell.execute_reply":"2022-02-25T00:31:37.622109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_RFM['Segment'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:31:37.624061Z","iopub.execute_input":"2022-02-25T00:31:37.624306Z","iopub.status.idle":"2022-02-25T00:31:37.670019Z","shell.execute_reply.started":"2022-02-25T00:31:37.624280Z","shell.execute_reply":"2022-02-25T00:31:37.669042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_RFM.reset_index(inplace=True)\ndf_RFM.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:31:37.671507Z","iopub.execute_input":"2022-02-25T00:31:37.672023Z","iopub.status.idle":"2022-02-25T00:31:37.704525Z","shell.execute_reply.started":"2022-02-25T00:31:37.671977Z","shell.execute_reply":"2022-02-25T00:31:37.703436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_Segment = df_RFM.groupby('Segment', as_index=False).agg({'Monetary':sum, 'card_id':groupby_count, 'Frequency':sum})\ndf_Segment.columns = ['Segment','Monetary', 'No_Cards', 'Frequency']\ndf_Segment","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:31:37.706211Z","iopub.execute_input":"2022-02-25T00:31:37.706535Z","iopub.status.idle":"2022-02-25T00:31:37.861074Z","shell.execute_reply.started":"2022-02-25T00:31:37.706495Z","shell.execute_reply":"2022-02-25T00:31:37.860324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_Loyal = df_RFM.groupby('IsLoyal', as_index=False).agg({'Monetary':sum, 'card_id':groupby_count, 'Frequency':sum})\ndf_Loyal.columns = ['Loyality','Monetary', 'No_Cards', 'Frequency']\ndf_Loyal","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:31:37.862305Z","iopub.execute_input":"2022-02-25T00:31:37.862905Z","iopub.status.idle":"2022-02-25T00:31:38.008972Z","shell.execute_reply.started":"2022-02-25T00:31:37.862874Z","shell.execute_reply":"2022-02-25T00:31:38.008213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"groups = df_Segment['Segment'].values.tolist()\namount = df_Segment['Monetary'].values.tolist()\n#colors = ['red', 'yellow', 'green', 'orange']\n\ntrace = go.Pie(labels=groups, values=amount, hoverinfo='label+percent', textinfo='value', textfont=dict(size=25),\n       pull=.4,hole=.2,marker=dict(line=dict(color='#000000', width=3)))\n\niplot([trace])","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:31:38.010265Z","iopub.execute_input":"2022-02-25T00:31:38.010487Z","iopub.status.idle":"2022-02-25T00:31:38.750313Z","shell.execute_reply.started":"2022-02-25T00:31:38.010463Z","shell.execute_reply":"2022-02-25T00:31:38.749207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"groups = df_Loyal['Loyality'].values.tolist()\namount = df_Loyal['Monetary'].values.tolist()\n#colors = ['red', 'yellow', 'green', 'orange']\n\ntrace = go.Pie(labels=groups, values=amount, hoverinfo='label+percent', textinfo='value', textfont=dict(size=25),\n       pull=.4,hole=.2,marker=dict(line=dict(color='#000000', width=3)))\n\niplot([trace])","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:31:38.751771Z","iopub.execute_input":"2022-02-25T00:31:38.752084Z","iopub.status.idle":"2022-02-25T00:31:38.793119Z","shell.execute_reply.started":"2022-02-25T00:31:38.752047Z","shell.execute_reply":"2022-02-25T00:31:38.792171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"groups = df_Segment['Segment'].values.tolist()\nNo_Company = df_Segment['No_Cards'].values.tolist()\nDistinct_Frequency = df_Segment['Frequency'].values.tolist()\n#colors = ['blue','red', 'yellow', 'pink','violet','green', 'orange']\n\n#trace2 = go.Bar(x=groups,y=No_Company,name='Companies', marker=dict(color=colors))\n\ntrace1 = go.Bar(x=groups,y=Distinct_Frequency,name='Frequency')\ntrace2 = go.Bar(x=groups,y=No_Company,name='Cards/Customers')\ndata = [trace1, trace2]\n\nlayout = go.Layout(barmode='stack')\n\nfig = go.Figure(data=data, layout=layout)\niplot(fig, filename='stacked-bar')","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:31:38.794469Z","iopub.execute_input":"2022-02-25T00:31:38.794761Z","iopub.status.idle":"2022-02-25T00:31:38.839404Z","shell.execute_reply.started":"2022-02-25T00:31:38.794722Z","shell.execute_reply":"2022-02-25T00:31:38.838468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"groups = df_Loyal['Loyality'].values.tolist()\nNo_Company = df_Loyal['No_Cards'].values.tolist()\nDistinct_Frequency = df_Loyal['Frequency'].values.tolist()\n#colors = ['blue','red', 'yellow', 'pink','violet','green', 'orange']\n\n#trace2 = go.Bar(x=groups,y=No_Company,name='Companies', marker=dict(color=colors))\n\ntrace1 = go.Bar(x=groups,y=Distinct_Frequency,name='Frequency')\ntrace2 = go.Bar(x=groups,y=No_Company,name='Cards/Customers')\ndata = [trace1, trace2]\n\nlayout = go.Layout(barmode='stack')\n\nfig = go.Figure(data=data, layout=layout)\niplot(fig, filename='stacked-bar')","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:31:38.840953Z","iopub.execute_input":"2022-02-25T00:31:38.841240Z","iopub.status.idle":"2022-02-25T00:31:38.878004Z","shell.execute_reply.started":"2022-02-25T00:31:38.841210Z","shell.execute_reply":"2022-02-25T00:31:38.877375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Clustering Algorithms Based Segmentation","metadata":{}},{"cell_type":"code","source":"# df_rfm.set_index('card_id', inplace=True)\n# df_rfm.head(1)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:31:38.879073Z","iopub.execute_input":"2022-02-25T00:31:38.879520Z","iopub.status.idle":"2022-02-25T00:31:38.883401Z","shell.execute_reply.started":"2022-02-25T00:31:38.879474Z","shell.execute_reply":"2022-02-25T00:31:38.882654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rank_df = df_RFM[['Recency','Frequency', 'Monetary']].rank(method='first')\nrank_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:31:38.892665Z","iopub.execute_input":"2022-02-25T00:31:38.893560Z","iopub.status.idle":"2022-02-25T00:31:39.080517Z","shell.execute_reply.started":"2022-02-25T00:31:38.893513Z","shell.execute_reply":"2022-02-25T00:31:39.079613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"normalized_df = (rank_df - rank_df.mean()) / rank_df.std()\nnormalized_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:31:39.082034Z","iopub.execute_input":"2022-02-25T00:31:39.082554Z","iopub.status.idle":"2022-02-25T00:31:39.104400Z","shell.execute_reply.started":"2022-02-25T00:31:39.082512Z","shell.execute_reply":"2022-02-25T00:31:39.103442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.cluster import KMeans\ndata = normalized_df[['Recency', 'Frequency', 'Monetary']]\n\nsse = {}\nfor k in range(1, 10):\n    kmeans = KMeans(n_clusters=k, max_iter=1000).fit(data)\n    data[\"clusters\"] = kmeans.labels_\n    sse[k] = kmeans.inertia_ # Inertia: Sum of distances of samples to their closest cluster center\nplt.figure()\nplt.plot(list(sse.keys()), list(sse.values()))\nplt.xlabel(\"Number of cluster\")\nplt.ylabel(\"SSE\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:31:39.105971Z","iopub.execute_input":"2022-02-25T00:31:39.106556Z","iopub.status.idle":"2022-02-25T00:32:11.139931Z","shell.execute_reply.started":"2022-02-25T00:31:39.106515Z","shell.execute_reply":"2022-02-25T00:32:11.139126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import silhouette_score","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:32:11.141866Z","iopub.execute_input":"2022-02-25T00:32:11.142497Z","iopub.status.idle":"2022-02-25T00:32:11.147674Z","shell.execute_reply.started":"2022-02-25T00:32:11.142445Z","shell.execute_reply":"2022-02-25T00:32:11.146664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(normalized_df.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:32:11.151709Z","iopub.execute_input":"2022-02-25T00:32:11.152376Z","iopub.status.idle":"2022-02-25T00:32:11.165011Z","shell.execute_reply.started":"2022-02-25T00:32:11.152329Z","shell.execute_reply":"2022-02-25T00:32:11.164266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.cluster import KMeans\n\nkmeans = KMeans(n_clusters=4,max_iter=1000).fit(normalized_df[['Recency', 'Frequency', 'Monetary']])\nprint(kmeans.labels_)\nprint(kmeans.cluster_centers_)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:32:11.166006Z","iopub.execute_input":"2022-02-25T00:32:11.166269Z","iopub.status.idle":"2022-02-25T00:32:14.599282Z","shell.execute_reply.started":"2022-02-25T00:32:11.166238Z","shell.execute_reply":"2022-02-25T00:32:14.598276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import Counter\nz = kmeans.labels_\nCounter(z)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:32:14.602900Z","iopub.execute_input":"2022-02-25T00:32:14.604997Z","iopub.status.idle":"2022-02-25T00:32:14.752927Z","shell.execute_reply.started":"2022-02-25T00:32:14.604956Z","shell.execute_reply":"2022-02-25T00:32:14.752207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"normalized_df['Cluster'] = kmeans.labels_\nnormalized_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:32:14.757306Z","iopub.execute_input":"2022-02-25T00:32:14.759493Z","iopub.status.idle":"2022-02-25T00:32:14.775898Z","shell.execute_reply.started":"2022-02-25T00:32:14.759310Z","shell.execute_reply":"2022-02-25T00:32:14.775163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_kmeans_RFM = df_RFM[['Recency','Frequency', 'Monetary']].copy()\ndf_kmeans_RFM['Cluster'] = kmeans.labels_\nprint(df_kmeans_RFM['Cluster'].value_counts())\ndf_kmeans_RFM.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:32:14.780204Z","iopub.execute_input":"2022-02-25T00:32:14.782127Z","iopub.status.idle":"2022-02-25T00:32:14.814025Z","shell.execute_reply.started":"2022-02-25T00:32:14.782084Z","shell.execute_reply":"2022-02-25T00:32:14.813038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colors = ['red', 'yellow', 'green', 'orange']\n\nfor i in ['Monetary']:\n    trace = go.Pie(labels=df_kmeans_RFM['Cluster'], values=df_kmeans_RFM[i], \n           hoverinfo='label+percent', textinfo='value', textfont=dict(size=15),\n           marker=dict(colors=colors, line=dict(color='#000000', width=3)))\n    iplot([trace])","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:32:14.817880Z","iopub.execute_input":"2022-02-25T00:32:14.819771Z","iopub.status.idle":"2022-02-25T00:32:17.972986Z","shell.execute_reply.started":"2022-02-25T00:32:14.819728Z","shell.execute_reply":"2022-02-25T00:32:17.972291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_kmeans_RFM.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:32:17.973985Z","iopub.execute_input":"2022-02-25T00:32:17.974492Z","iopub.status.idle":"2022-02-25T00:32:17.987532Z","shell.execute_reply.started":"2022-02-25T00:32:17.974457Z","shell.execute_reply":"2022-02-25T00:32:17.985680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in [\"Recency\",\"Frequency\",\"Monetary\"]:\n    print()\n    plt.figure(figsize=(14,8))\n    ax = sns.boxplot(x=\"Cluster\", y=col, data=normalized_df)\n    plt.title('Cluster/Segment wise Difference in Purchase '+col,fontsize=15)\n    plt.show()\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:32:17.989022Z","iopub.execute_input":"2022-02-25T00:32:17.989384Z","iopub.status.idle":"2022-02-25T00:32:18.922907Z","shell.execute_reply.started":"2022-02-25T00:32:17.989343Z","shell.execute_reply":"2022-02-25T00:32:18.921973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ********** CLV Prediction using RFM**************","metadata":{}},{"cell_type":"markdown","source":"For the CLV models, the following components are used:\n\n* Recency - This represents the age of the customer when they made their latest transactions. (Current_date - last_transaction_date)\n* Frequency - This represents the total number of transactions/number of visits a customer has made. (Count of total transactions)\n* Monetary - This represents the total purchase amount that a specified customer has made. (Sum of purchase_amt)\n* Time - This represents the age of the customer. Time span between a customer’s first and last transaction.","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\n\n# hist = pd.read_csv('../input/elo-merchant-category-recommendation/historical_transactions.csv')","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:32:18.924398Z","iopub.execute_input":"2022-02-25T00:32:18.924678Z","iopub.status.idle":"2022-02-25T00:32:18.929276Z","shell.execute_reply.started":"2022-02-25T00:32:18.924637Z","shell.execute_reply":"2022-02-25T00:32:18.928229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_historical = df_historical[['card_id','purchase_date','purchase_amount']]\ndf_historical = df_historical.sort_values(by=['card_id', 'purchase_date'], ascending=[True, True])","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:32:18.930366Z","iopub.execute_input":"2022-02-25T00:32:18.930620Z","iopub.status.idle":"2022-02-25T00:32:58.476142Z","shell.execute_reply.started":"2022-02-25T00:32:18.930590Z","shell.execute_reply":"2022-02-25T00:32:58.475242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_historical.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:32:58.477561Z","iopub.execute_input":"2022-02-25T00:32:58.477775Z","iopub.status.idle":"2022-02-25T00:32:58.488589Z","shell.execute_reply.started":"2022-02-25T00:32:58.477750Z","shell.execute_reply":"2022-02-25T00:32:58.487695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Time\nfrom datetime import datetime\n\nz = df_historical.groupby('card_id')['purchase_date'].max().reset_index()\nq = df_historical.groupby('card_id')['purchase_date'].min().reset_index()\n\nz.columns = ['card_id', 'Max']\nq.columns = ['card_id', 'Min']\n\n## Extracting current timestamp\nnow = datetime.now()\ncurr_date = now.strftime(\"%m-%d-%Y, %H:%M:%S\")\ncurr_date = pd.to_datetime(curr_date)\n\nrec = pd.merge(z,q,how = 'left',on = 'card_id')\nrec['Min'] = pd.to_datetime(rec['Min'])\nrec['Max'] = pd.to_datetime(rec['Max'])\n\n## Time value \nrec['Recency'] = (curr_date - rec['Max']).astype('timedelta64[D]') ## current date - most recent date\n\n## Recency value\nrec['Time'] = (rec['Max'] - rec['Min']).astype('timedelta64[D]') ## Age of customer, MAX - MIN\n\nrec = rec[['card_id','Time','Recency']]\nrec.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:32:58.490413Z","iopub.execute_input":"2022-02-25T00:32:58.490623Z","iopub.status.idle":"2022-02-25T00:33:05.717904Z","shell.execute_reply.started":"2022-02-25T00:32:58.490599Z","shell.execute_reply":"2022-02-25T00:33:05.717100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Frequency\nfreq = df_historical.groupby('card_id').size().reset_index()\nfreq.columns = ['card_id', 'Frequency']\nfreq.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:33:05.719055Z","iopub.execute_input":"2022-02-25T00:33:05.719873Z","iopub.status.idle":"2022-02-25T00:33:09.165283Z","shell.execute_reply.started":"2022-02-25T00:33:05.719840Z","shell.execute_reply":"2022-02-25T00:33:09.164360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Monetary\nmon = df_historical.groupby('card_id')['purchase_amount'].sum().reset_index()\nmon.columns = ['card_id', 'Monitary']\nmon.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:33:09.166630Z","iopub.execute_input":"2022-02-25T00:33:09.166939Z","iopub.status.idle":"2022-02-25T00:33:12.647493Z","shell.execute_reply.started":"2022-02-25T00:33:09.166902Z","shell.execute_reply":"2022-02-25T00:33:12.646815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final = pd.merge(freq,mon,how = 'left', on = 'card_id')\nfinal = pd.merge(final,rec,how = 'left', on = 'card_id')\n\nfinal['historic_CLV'] = final['Frequency'] * final['Monitary'] \nfinal['AOV'] = final['Monitary']/final['Frequency'] ## AOV - Average order value (i.e) total_purchase_amt/total_trans\nfinal['Predictive_CLV'] = final['Time']*final['AOV']*final['Monitary']*final['Recency'] \n\nfinal.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:33:12.648438Z","iopub.execute_input":"2022-02-25T00:33:12.648682Z","iopub.status.idle":"2022-02-25T00:33:13.376056Z","shell.execute_reply.started":"2022-02-25T00:33:12.648645Z","shell.execute_reply":"2022-02-25T00:33:13.375220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Above features may boost our model performance","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ****************Some EDA with the Date Variables*************","metadata":{}},{"cell_type":"code","source":"import warnings\nimport datetime\nimport calendar\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom datetime import time\nfrom dateutil.relativedelta import relativedelta\n\n# to ignore future warnings\nwarnings.simplefilter(action = 'ignore', category = FutureWarning)\n\n# set size of seaborn plots\nsns.set(rc={'figure.figsize':(10, 7)})","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:33:13.377494Z","iopub.execute_input":"2022-02-25T00:33:13.377819Z","iopub.status.idle":"2022-02-25T00:33:13.385472Z","shell.execute_reply.started":"2022-02-25T00:33:13.377787Z","shell.execute_reply":"2022-02-25T00:33:13.384395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/elo-merchant-category-recommendation/train.csv', sep = ',')\ntest = pd.read_csv('../input/elo-merchant-category-recommendation/test.csv', sep = ',')\nmerchants = pd.read_csv('../input/elo-merchant-category-recommendation/merchants.csv', sep = ',')\nnew_merchant = pd.read_csv('../input/elo-merchant-category-recommendation/new_merchant_transactions.csv', sep = ',')","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:33:13.386771Z","iopub.execute_input":"2022-02-25T00:33:13.387357Z","iopub.status.idle":"2022-02-25T00:33:19.850191Z","shell.execute_reply.started":"2022-02-25T00:33:13.387326Z","shell.execute_reply":"2022-02-25T00:33:19.849287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_train = reduce_mem_usage(pd.read_csv(\"../input/elo-merchant-category-recommendation/train.csv\"))\n#df_test = reduce_mem_usage(pd.read_csv(\"../input/elo-merchant-category-recommendation/test.csv\"))\n#df_historical =reduce_mem_usage(pd.read_csv(\"../input/elo-merchant-category-recommendation/historical_transactions.csv\",parse_dates=['purchase_date']))\n#df_new =reduce_mem_usage(pd.read_csv(\"../input/elo-merchant-category-recommendation/new_merchant_transactions.csv\",parse_dates=['purchase_date']))","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:33:19.851522Z","iopub.execute_input":"2022-02-25T00:33:19.851822Z","iopub.status.idle":"2022-02-25T00:33:19.856051Z","shell.execute_reply.started":"2022-02-25T00:33:19.851786Z","shell.execute_reply":"2022-02-25T00:33:19.855254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train = df_train\n#test = df_test\n#merchants = pd.read_csv('../input/elo-merchant-category-recommendation/merchants.csv', sep = ',')\n#new_merchant = ","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:33:19.857525Z","iopub.execute_input":"2022-02-25T00:33:19.857977Z","iopub.status.idle":"2022-02-25T00:33:19.870396Z","shell.execute_reply.started":"2022-02-25T00:33:19.857936Z","shell.execute_reply":"2022-02-25T00:33:19.869504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## prepare data\n# this is not a valid approach if you want to build models from the data\n# drop some redundant columns\ndropping = ['merchant_category_id', 'subsector_id', 'category_1', 'city_id', 'state_id',\n            'category_2']\nfor var in dropping:\n    merchants = merchants.drop(var, axis = 1)\n\n# merge merchants with new_merchants\ndata = pd.merge(merchants, new_merchant, on = 'merchant_id')\n\n# merge data with train data\ndata = pd.merge(data, train, on = 'card_id')","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:33:19.871679Z","iopub.execute_input":"2022-02-25T00:33:19.871924Z","iopub.status.idle":"2022-02-25T00:33:26.732629Z","shell.execute_reply.started":"2022-02-25T00:33:19.871896Z","shell.execute_reply":"2022-02-25T00:33:26.731677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The variables in question here are first_active_month and purchase_time. Let's take a brief look at their number of unique values and the first five values:","metadata":{}},{"cell_type":"code","source":"print(len(data['first_active_month'].unique()))\ndata['first_active_month'][:5]","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:33:26.734151Z","iopub.execute_input":"2022-02-25T00:33:26.734394Z","iopub.status.idle":"2022-02-25T00:33:26.840113Z","shell.execute_reply.started":"2022-02-25T00:33:26.734365Z","shell.execute_reply":"2022-02-25T00:33:26.839228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(data['purchase_date'].unique()))\ndata['purchase_date'][:5]","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:33:26.841831Z","iopub.execute_input":"2022-02-25T00:33:26.842180Z","iopub.status.idle":"2022-02-25T00:33:27.426278Z","shell.execute_reply.started":"2022-02-25T00:33:26.842107Z","shell.execute_reply":"2022-02-25T00:33:27.425489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#This shows us that the purchase_date variable actually carrys more information than it's name makes it sound like. It's not only the date, but also the specific time. Let's recode this into two variables:","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:33:27.427372Z","iopub.execute_input":"2022-02-25T00:33:27.427659Z","iopub.status.idle":"2022-02-25T00:33:27.432739Z","shell.execute_reply.started":"2022-02-25T00:33:27.427624Z","shell.execute_reply":"2022-02-25T00:33:27.431714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# recode purchase_date\ndata['purchase_time'] = data['purchase_date'].str.split(' ')\ndata['purchase_date'] = data['purchase_time'].str[0]\ndata['purchase_time'] = data['purchase_time'].str[1]","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:33:27.434082Z","iopub.execute_input":"2022-02-25T00:33:27.434469Z","iopub.status.idle":"2022-02-25T00:33:32.441357Z","shell.execute_reply.started":"2022-02-25T00:33:27.434431Z","shell.execute_reply":"2022-02-25T00:33:32.440559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now there are two different strategies to use these time variables in further models.\n\nRecode them to a linear variable where the lowest number is the day furthest in the past and the highest number the most recent day\nRecode them to ordered categorical variables\nLet's start with number 1:","metadata":{}},{"cell_type":"code","source":"def dates_to_numeric(series, kind = 'month'):\n    # get all unique values\n    months = list(series.unique())\n\n    # sort them\n    if kind == 'month':\n        date_string = \"%Y-%m\"\n    elif kind == 'day':\n        date_string = \"%Y-%m-%d\"\n\n    # make them a datetime object\n    dates = [datetime.datetime.strptime(ts, date_string) for ts in months]\n    dates.sort()\n    sorteddates = [datetime.datetime.strftime(ts, date_string) for ts in dates]\n\n    # generate all month stamps between first and last\n    start_date = sorteddates[0]\n    end_date = sorteddates[len(sorteddates) - 1]\n    \n    cur_date = start = datetime.datetime.strptime(start_date, date_string).date()\n    end = datetime.datetime.strptime(end_date, date_string).date()\n\n    months = []\n    while cur_date < end:\n        if kind == 'month':\n            months.append(str(cur_date)[:-3])\n            cur_date += relativedelta(months = 1)\n        elif kind == 'day':\n            months.append(str(cur_date))\n            cur_date += relativedelta(days = 1)\n    \n    # create dict that maps new values to each month\n    map_dict = {}\n    keys = range(0, len(months))\n    for i in keys:\n        map_dict[i] = months[i]\n\n    # reverse dict keys / values for mapping\n    new_dict = {v: k for k, v in map_dict.items()}\n    return new_dict\n\nnew_dict = dates_to_numeric(data['first_active_month'])\ndata['first_active_month_numeric'] = data['first_active_month'].apply(lambda x: new_dict.get(x))\n\nnew_dict = dates_to_numeric(data['purchase_date'], kind = 'day')\ndata['purchase_date_numeric'] = data['purchase_date'].apply(lambda x: new_dict.get(x))\n\n# recode timestamp to number of seconds passed since 00:00:00\ndef timestamp_to_seconds(time):\n    seconds = sum(x * int(t) for x, t in zip([3600, 60, 1], time.split(':'))) \n    return seconds\n\ndata['purchase_seconds'] = data['purchase_time'].apply(lambda x: timestamp_to_seconds(x))","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:33:32.443027Z","iopub.execute_input":"2022-02-25T00:33:32.443699Z","iopub.status.idle":"2022-02-25T00:33:37.371359Z","shell.execute_reply.started":"2022-02-25T00:33:32.443655Z","shell.execute_reply":"2022-02-25T00:33:37.370531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = sns.regplot(x = data['first_active_month_numeric'], y = data['target'], marker = \"+\",\n                 lowess = True, line_kws = {'color': 'black'})\nax.set_title('Relationship of the target variable and linear first active month')\nax.set_xlabel('first active month linear')","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:33:37.372736Z","iopub.execute_input":"2022-02-25T00:33:37.373233Z","iopub.status.idle":"2022-02-25T00:33:45.578501Z","shell.execute_reply.started":"2022-02-25T00:33:37.373203Z","shell.execute_reply":"2022-02-25T00:33:45.577658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = sns.regplot(x = data['purchase_date_numeric'], y = data['target'], marker = \"+\",\n                 lowess = True, line_kws = {'color': 'black'})\nax.set_title('Relationship of the target variable and linear purchase date')\nax.set_xlabel('purchase date linear')","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:33:45.579747Z","iopub.execute_input":"2022-02-25T00:33:45.580000Z","iopub.status.idle":"2022-02-25T00:34:20.672117Z","shell.execute_reply.started":"2022-02-25T00:33:45.579969Z","shell.execute_reply":"2022-02-25T00:34:20.671401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ax = sns.regplot(x = data['purchase_seconds'], y = data['target'], marker = \"+\",\n                 #lowess = True, line_kws = {'color': 'black'})\n#ax.set_xlabel('purchase seconds linear')\n# This takes incredibly long and is therefore commented out. It however looks very similar to the plot above","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:34:20.673253Z","iopub.execute_input":"2022-02-25T00:34:20.673494Z","iopub.status.idle":"2022-02-25T00:34:20.678513Z","shell.execute_reply.started":"2022-02-25T00:34:20.673467Z","shell.execute_reply":"2022-02-25T00:34:20.677228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"These plot show no relationship between the metric versions of the three date variables and the target variable. By using the lowess smoother instead of a linear regression we also made sure there is no nonlinear relationship between the two variables.\n\nNow that this is out of the way, let's dive into the more reasonable data transformations. For the purchase_date variable we will create a new variable that contains the name of the corresponding weekday (e.g. monday, tuesday ..). For the purchase_time variable we will create a new categorical variable with 4 categories: Morning, Afternoon, Evening and Night. These correspond to:\n\nMorning: 5am to 12pm (05:00 to 11:59)\n\nAfternoon: 12pm to 5pm (12:00 to 16:59)\n\nEvening: 5pm to 9pm (17:00 to 20:59)\n\nNight: 9pm to 5am (21:00 to 04:59)\n\nWe will also create two variables containing the corresponding month (e.g. January, February ..) for purchase_date and first_active_month. For the latter we will create a first_active_year variable as well. **Let's start!\n\nEDIT: I forgot that the time of the month itself (beginning, end etc.) may have an effect on the target variable as well. This will be explored too in this second version. We will look both at a categorical version and a numeric version just using the day.","metadata":{}},{"cell_type":"code","source":"def get_weekday(date_string):\n    date = datetime.datetime.strptime(date_string, '%Y-%m-%d')\n    return calendar.day_name[date.weekday()]\n\n# get weekday for date variable\ndata['purchase_weekday'] = data['purchase_date'].apply(lambda x: get_weekday(x))\n\n# for plotting recode to ordered categorical\nday_labels = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']\ndata['purchase_weekday'] = pd.Categorical(data['purchase_weekday'], categories = day_labels, \n                                          ordered = True)\n\ndef get_month(date_string, kind = 'month'):\n    if kind == 'month':\n        date = datetime.datetime.strptime(date_string, '%Y-%m')\n    elif kind == 'day':\n        date = datetime.datetime.strptime(date_string, '%Y-%m-%d')\n    return date.strftime(\"%B\")\n\ndata['purchase_month'] = data['purchase_date'].apply(lambda x: get_month(x, kind = 'day'))\ndata['first_active_month2'] = data['first_active_month'].apply(lambda x: get_month(x))\ndata['first_active_year'] = data['first_active_month'].str[:4]\n\nmonth_labels = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August',\n                'September', 'October', 'November', 'December']\ndata['purchase_month'] = pd.Categorical(data['purchase_month'], categories = month_labels, \n                                          ordered = True)\ndata['first_active_month2'] = pd.Categorical(data['first_active_month2'], categories = month_labels, \n                                          ordered = True)\n\nyear_labels = ['2011', '2012', '2013', '2014', '2015', '2016', '2017', '2018']\ndata['first_active_year'] = pd.Categorical(data['first_active_year'], categories = year_labels, \n                                          ordered = True)\n\n# get time of the day\ndata['temp'] = data['purchase_time'].str.split(':')\n\ndef get_session(time_list):\n    time_list[0] = int(time_list[0])\n    if time_list[0] > 4 and time_list[0] < 12:\n        return 'Morning'\n    elif time_list[0] >= 12 and time_list[0] < 17:\n        return 'Afternoon'\n    elif time_list[0] >= 17 and time_list[0] < 21:\n        return 'Evening'\n    else:\n        return 'Night'\n    \ndata['purchase_session'] = data['temp'].apply(lambda x: get_session(x))\n\nsession_labels = ['Morning', 'Afternoon', 'Evening', 'Night']\ndata['purchase_session'] = pd.Categorical(data['purchase_session'], categories = session_labels, \n                                          ordered = True)\n## time of month\n# as categorical variable, thressholds are arbitrary and could be different\ndef get_time_of_month_cat(date):\n    date_temp = date.split('-')\n    if int(date_temp[2]) < 10:\n        time_of_month = 'Beginning'\n    elif int(date_temp[2]) >= 10 and int(date_temp[2]) < 20:\n        time_of_month = 'Middle'\n    else:\n        time_of_month = 'End'\n    return time_of_month\n\ndata['time_of_month_cat'] = data['purchase_date'].apply(lambda x: get_time_of_month_cat(x))\n\ntof_labels = ['Beginning', 'Middle', 'End']\ndata['time_of_month_cat'] = pd.Categorical(data['time_of_month_cat'], categories = tof_labels, \n                                           ordered = True)\n\ndata['time_of_month_num'] = data['purchase_date'].str[8:].astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:34:20.680030Z","iopub.execute_input":"2022-02-25T00:34:20.680352Z","iopub.status.idle":"2022-02-25T00:35:27.963106Z","shell.execute_reply.started":"2022-02-25T00:34:20.680312Z","shell.execute_reply":"2022-02-25T00:35:27.962267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = sns.lineplot(x = \"purchase_weekday\", y = \"target\", \n                  markers = True, dashes = False, data = data)\nplt.xticks(rotation = 45)\nax.set_title('Target Variable Changes over Purchase Week')\nax.set_xlabel('Purchase Weekday')","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:35:27.964386Z","iopub.execute_input":"2022-02-25T00:35:27.964609Z","iopub.status.idle":"2022-02-25T00:35:37.985237Z","shell.execute_reply.started":"2022-02-25T00:35:27.964584Z","shell.execute_reply":"2022-02-25T00:35:37.984498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is a pattern here. The target variable follows a non-linear curve over the weekdays. The differences may be small, but they are statistically significant.","metadata":{}},{"cell_type":"code","source":"ax = sns.lineplot(x = \"purchase_month\", y = \"target\", \n                  markers = True, dashes = False, data = data)\nplt.xticks(rotation = 45)\nax.set_title('Target Variable Changes over Purchase Month')\nax.set_xlabel('Purchase Month')","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:35:37.986512Z","iopub.execute_input":"2022-02-25T00:35:37.986930Z","iopub.status.idle":"2022-02-25T00:35:49.923967Z","shell.execute_reply.started":"2022-02-25T00:35:37.986886Z","shell.execute_reply":"2022-02-25T00:35:49.923110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now this is more like it! There are pretty big differences in the mean of the target variable between each month of purchase. In January the mean is close to - 2 while in April it's at - 0,25. Adding the purchase month as dummy variables to your model looks promising.","metadata":{}},{"cell_type":"code","source":"ax = sns.lineplot(x = \"first_active_month2\", y = \"target\", \n                  markers = True, dashes = False, data = data)\nplt.xticks(rotation = 45)\nax.set_title('Target Variable Changes over the First Active Month')\nax.set_xlabel('First Active Month')","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:35:49.925248Z","iopub.execute_input":"2022-02-25T00:35:49.926181Z","iopub.status.idle":"2022-02-25T00:35:59.441224Z","shell.execute_reply.started":"2022-02-25T00:35:49.926147Z","shell.execute_reply":"2022-02-25T00:35:59.440220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are some clear variations in this plot as well, which follow almost a step-wise linear pattern. The differences might not be nearly as strong as in the purchase month plot, but this still might help improve your model.","metadata":{}},{"cell_type":"code","source":"ax = sns.lineplot(x = \"first_active_year\", y = \"target\", \n                  markers = True, dashes = False, data = data)\nplt.xticks(rotation = 45)\nax.set_title('Target Variable Changes over the First Active Year')\nax.set_xlabel('First Active Year')","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:35:59.442607Z","iopub.execute_input":"2022-02-25T00:35:59.442867Z","iopub.status.idle":"2022-02-25T00:36:14.379807Z","shell.execute_reply.started":"2022-02-25T00:35:59.442839Z","shell.execute_reply":"2022-02-25T00:36:14.378877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The first active year show some big differences! The target variable increases with each year. Definitly an interesting pattern.","metadata":{}},{"cell_type":"code","source":"ax = sns.lineplot(x = \"purchase_session\", y = \"target\", \n                  markers = True, dashes = False, data = data)\nplt.xticks(rotation = 45)\nax.set_title('Target Variable Changes over Purchase Time of Day')\nax.set_xlabel('Purchase Time of Day')","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:36:14.381107Z","iopub.execute_input":"2022-02-25T00:36:14.381386Z","iopub.status.idle":"2022-02-25T00:36:25.796964Z","shell.execute_reply.started":"2022-02-25T00:36:14.381358Z","shell.execute_reply":"2022-02-25T00:36:25.796043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Differences over the day are rather small, but follow a clear pattern as well. Let's see if that pattern holds up if we look at it by different week days:","metadata":{}},{"cell_type":"code","source":"ax = sns.catplot(x = 'purchase_weekday', y = 'target', hue = 'purchase_session', data = data,\n                kind = 'bar', height = 5, aspect = 2)\nax.despine(left = True)\nplt.xticks(rotation = 45)\nax.set_ylabels(\"target\")\nax.set_xlabels('Weekday')","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:36:25.798227Z","iopub.execute_input":"2022-02-25T00:36:25.798467Z","iopub.status.idle":"2022-02-25T00:36:38.942632Z","shell.execute_reply.started":"2022-02-25T00:36:25.798439Z","shell.execute_reply":"2022-02-25T00:36:38.940836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It seems like it does. The pattern seems to be nearly the same on all weekdays. There are some differences between saturday and tuesday though.\n\nAll in all there are some interesting relationships in the date variables of this data set","metadata":{}},{"cell_type":"code","source":"ax = sns.regplot(x = data['time_of_month_num'], y = data['target'], marker = \"+\",\n                 lowess = True, line_kws = {'color': 'black'})\nax.set_title('Relationship of the target variable and purchase time of month')\nax.set_xlabel('time of purchase inside month')","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:36:38.944043Z","iopub.execute_input":"2022-02-25T00:36:38.944832Z","iopub.status.idle":"2022-02-25T00:36:43.827087Z","shell.execute_reply.started":"2022-02-25T00:36:38.944788Z","shell.execute_reply":"2022-02-25T00:36:43.826208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Using just the day itself doesn't seem very useful. No direct relationship can be observed. Now let's look at the categorical version:","metadata":{}},{"cell_type":"code","source":"ax = sns.lineplot(x = \"time_of_month_cat\", y = \"target\", \n                  markers = True, dashes = False, data = data)\nplt.xticks(rotation = 45)\nax.set_title('Target Variable Changes over Purchase Time of Month')\nax.set_xlabel('Purchase Time of Month')","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:36:43.828422Z","iopub.execute_input":"2022-02-25T00:36:43.828666Z","iopub.status.idle":"2022-02-25T00:36:55.331414Z","shell.execute_reply.started":"2022-02-25T00:36:43.828637Z","shell.execute_reply":"2022-02-25T00:36:55.330406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The pattern is really small with the biggest deviation being between -0.52 and -0.66, but it does exist. I am not sure if this will be particularly useful but it's definitly worth a try. Let's also look if this pattern is the same in each month:","metadata":{}},{"cell_type":"code","source":"ax = sns.catplot(x = 'purchase_month', y = 'target', hue = 'time_of_month_cat', data = data,\n                kind = 'bar', height = 5, aspect = 2)\nax.despine(left = True)\nplt.xticks(rotation = 45)\nax.set_ylabels(\"Target\")\nax.set_xlabels('Purchase Time of Month')","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:36:55.332638Z","iopub.execute_input":"2022-02-25T00:36:55.332857Z","iopub.status.idle":"2022-02-25T00:37:08.392692Z","shell.execute_reply.started":"2022-02-25T00:36:55.332832Z","shell.execute_reply":"2022-02-25T00:37:08.391706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The pattern obviously isn't really stable at all. The only consistent finding is, that the end of the month seemingly always has a higher score on the target variable then other parts. If only the very end of the month has an effect on the target variable, we might still be able to utilize this. Let's try by creating a dummy variable that only looks at the very last days of the month:","metadata":{}},{"cell_type":"code","source":"def get_end_of_month(date):\n    date_temp = date.split('-')\n    if int(date_temp[2]) >= 25:\n        end_of_month = 'Yes'\n    else:\n        end_of_month = 'No'\n    return end_of_month\n\ndata['end_of_month'] = data['purchase_date'].apply(lambda x: get_end_of_month(x))\n\nax = sns.barplot(x = 'end_of_month', y = 'target', data = data)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:37:08.394164Z","iopub.execute_input":"2022-02-25T00:37:08.394410Z","iopub.status.idle":"2022-02-25T00:37:24.613968Z","shell.execute_reply.started":"2022-02-25T00:37:08.394383Z","shell.execute_reply":"2022-02-25T00:37:24.613139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This shows bigger differences than before.","metadata":{}},{"cell_type":"markdown","source":"# ***********Implementing Models *********************","metadata":{}},{"cell_type":"code","source":"new_transactions = df_new#pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/new_merchant_transactions.csv',\n                               #parse_dates=['purchase_date'])\n\nhistorical_transactions = df_historical#pd.read_csv('/kaggle/input/elo-merchant-category-recommendation/historical_transactions.csv',\n                                      #parse_dates=['purchase_date'])\n\ndef read_data(input_file):\n    df = pd.read_csv(input_file)\n    df['first_active_month'] = pd.to_datetime(df['first_active_month'])\n    df['elapsed_time'] = (datetime.date(2018, 2, 1) - df['first_active_month'].dt.date).dt.days\n    return df\n#_________________________________________\ntrain = df_train#read_data('/kaggle/input/elo-merchant-category-recommendation/train.csv')\ntest = df_test#read_data('/kaggle/input/elo-merchant-category-recommendation/test.csv')\n\ntarget = train['target']\nprint(target.shape)\nprint(test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:48:13.437923Z","iopub.execute_input":"2022-02-25T00:48:13.438976Z","iopub.status.idle":"2022-02-25T00:48:13.449361Z","shell.execute_reply.started":"2022-02-25T00:48:13.438926Z","shell.execute_reply":"2022-02-25T00:48:13.448315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_train = reduce_mem_usage(pd.read_csv(\"../input/elo-merchant-category-recommendation/train.csv\"))\n#df_test = reduce_mem_usage(pd.read_csv(\"../input/elo-merchant-category-recommendation/test.csv\"))\n#df_historical =reduce_mem_usage(pd.read_csv(\"../input/elo-merchant-category-recommendation/historical_transactions.csv\",parse_dates=['purchase_date']))\n#df_new =reduce_mem_usage(pd.read_csv(\"../input/elo-merchant-category-recommendation/new_merchant_transactions.csv\",parse_dates=['purchase_date']))","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:48:13.451352Z","iopub.execute_input":"2022-02-25T00:48:13.451677Z","iopub.status.idle":"2022-02-25T00:48:13.465673Z","shell.execute_reply.started":"2022-02-25T00:48:13.451646Z","shell.execute_reply":"2022-02-25T00:48:13.464493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"historical_transactions.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:48:13.467308Z","iopub.execute_input":"2022-02-25T00:48:13.467701Z","iopub.status.idle":"2022-02-25T00:48:13.504064Z","shell.execute_reply.started":"2022-02-25T00:48:13.467670Z","shell.execute_reply":"2022-02-25T00:48:13.503085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Preprocessing**","metadata":{}},{"cell_type":"markdown","source":"One Hot Encoding\n\n","metadata":{}},{"cell_type":"code","source":"historical_transactions = pd.get_dummies(df_historical, columns=['category_2', 'category_3'])\nnew_transactions = pd.get_dummies(new_transactions, columns=['category_2', 'category_3'])","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:48:13.505967Z","iopub.execute_input":"2022-02-25T00:48:13.506475Z","iopub.status.idle":"2022-02-25T00:48:23.890439Z","shell.execute_reply.started":"2022-02-25T00:48:13.506428Z","shell.execute_reply":"2022-02-25T00:48:23.889445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Date processing\n\n","metadata":{}},{"cell_type":"code","source":"for df in [historical_transactions, new_transactions]:\n    df['purchase_date'] = pd.to_datetime(df['purchase_date'])\n    df['year'] = df['purchase_date'].dt.year\n    df['weekofyear'] = df['purchase_date'].dt.weekofyear\n    df['month'] = df['purchase_date'].dt.month\n    df['dayofweek'] = df['purchase_date'].dt.dayofweek\n    df['weekend'] = (df.purchase_date.dt.weekday >=5).astype(int)\n    df['hour'] = df['purchase_date'].dt.hour\n    df['authorized_flag'] = df['authorized_flag'].map({'Y':1, 'N':0})\n    df['category_1'] = df['category_1'].map({'Y':1, 'N':0}) \n    #https://www.kaggle.com/c/elo-merchant-category-recommendation/discussion/73244\n    df['month_diff'] = ((dt.datetime.today() - df['purchase_date']).dt.days)//30\n    df['month_diff'] += df['month_lag']","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:48:23.892553Z","iopub.execute_input":"2022-02-25T00:48:23.892896Z","iopub.status.idle":"2022-02-25T00:49:03.670537Z","shell.execute_reply.started":"2022-02-25T00:48:23.892867Z","shell.execute_reply":"2022-02-25T00:49:03.669642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Reduce memory usage","metadata":{}},{"cell_type":"code","source":"historical_transactions = reduce_mem_usage(historical_transactions)\nnew_transactions = reduce_mem_usage(new_transactions)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:49:03.671769Z","iopub.execute_input":"2022-02-25T00:49:03.672040Z","iopub.status.idle":"2022-02-25T00:49:05.050883Z","shell.execute_reply.started":"2022-02-25T00:49:03.672011Z","shell.execute_reply":"2022-02-25T00:49:05.050094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Feature Engineering\n","metadata":{}},{"cell_type":"markdown","source":"helper function to apply aggregations on existing features to create new features\n\n","metadata":{}},{"cell_type":"code","source":"def aggregate_transactions(history):\n    \n    history.loc[:, 'purchase_date'] = pd.DatetimeIndex(history['purchase_date']).\\\n                                      astype(np.int64) * 1e-9\n    \n    agg_func = {\n    'authorized_flag': ['mean'],\n    'category_1': ['sum', 'mean'],\n    'category_2_1.0': ['mean'],\n    'category_2_2.0': ['mean'],\n    'category_2_3.0': ['mean'],\n    'category_2_4.0': ['mean'],\n    'category_2_5.0': ['mean'],\n    'category_3_A': ['mean'],\n    'category_3_B': ['mean'],\n    'category_3_C': ['mean'],\n    'merchant_id': ['nunique'],\n    'merchant_category_id': ['nunique'],\n    'state_id': ['nunique'],\n    'city_id': ['nunique'],\n    'subsector_id': ['nunique'],\n    'purchase_amount': ['sum', 'mean', 'max', 'min', 'std'],\n    'installments': ['sum', 'mean', 'max', 'min', 'std'],\n    'purchase_date': [np.ptp, 'min', 'max'],\n    'month_lag': ['mean', 'max', 'min', 'std'],\n    'month_diff': ['mean'],\n    'month': ['nunique'],\n    'hour': ['nunique'],\n    'weekofyear': ['nunique'],\n    'dayofweek': ['nunique'],\n    'year': ['nunique'],\n    'authorized_flag': ['sum', 'mean'],\n    'weekend': ['sum', 'mean']\n    }\n    \n    agg_history = history.groupby(['card_id']).agg(agg_func)\n    agg_history.columns = ['_'.join(col).strip() for col in agg_history.columns.values]\n    agg_history.reset_index(inplace=True)\n    \n    df = (history.groupby('card_id')\n          .size()\n          .reset_index(name='transactions_count'))\n    \n    agg_history = pd.merge(df, agg_history, on='card_id', how='left')\n    \n    return agg_history","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:49:05.052351Z","iopub.execute_input":"2022-02-25T00:49:05.052779Z","iopub.status.idle":"2022-02-25T00:49:05.065329Z","shell.execute_reply.started":"2022-02-25T00:49:05.052721Z","shell.execute_reply":"2022-02-25T00:49:05.064639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"history stores aggregated results from historical transactions\n\n","metadata":{}},{"cell_type":"code","source":"history = aggregate_transactions(historical_transactions)\nhistory.columns = ['hist_' + c if c != 'card_id' else c for c in history.columns]\nhistory[:5]","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:49:05.066358Z","iopub.execute_input":"2022-02-25T00:49:05.066867Z","iopub.status.idle":"2022-02-25T00:51:20.333372Z","shell.execute_reply.started":"2022-02-25T00:49:05.066828Z","shell.execute_reply":"2022-02-25T00:51:20.332516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"new stores aggregated results from new merchant transactions\n\n","metadata":{}},{"cell_type":"code","source":"new = aggregate_transactions(new_transactions)\nnew.columns = ['new_' + c if c != 'card_id' else c for c in new.columns]\nnew[:5]","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:51:20.334526Z","iopub.execute_input":"2022-02-25T00:51:20.334748Z","iopub.status.idle":"2022-02-25T00:51:59.584627Z","shell.execute_reply.started":"2022-02-25T00:51:20.334714Z","shell.execute_reply":"2022-02-25T00:51:59.583997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Combine dataframes to train and test dataframes**\n\njoining datasets on the common id, card_id for both train and test\n\n","metadata":{}},{"cell_type":"code","source":"train = pd.merge(train, history, on='card_id', how='left')\ntest = pd.merge(test, history, on='card_id', how='left')\n\ntrain = pd.merge(train, new, on='card_id', how='left')\ntest = pd.merge(test, new, on='card_id', how='left')\n\nhistory[0::5]","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:51:59.585906Z","iopub.execute_input":"2022-02-25T00:51:59.586278Z","iopub.status.idle":"2022-02-25T00:52:01.900077Z","shell.execute_reply.started":"2022-02-25T00:51:59.586239Z","shell.execute_reply":"2022-02-25T00:52:01.899062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Impute missing values\n\n","metadata":{}},{"cell_type":"code","source":"from sklearn.impute import SimpleImputer\nmy_imputer = SimpleImputer()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:52:01.901233Z","iopub.execute_input":"2022-02-25T00:52:01.901472Z","iopub.status.idle":"2022-02-25T00:52:02.148548Z","shell.execute_reply.started":"2022-02-25T00:52:01.901444Z","shell.execute_reply":"2022-02-25T00:52:02.147588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get features and remove any that have the incorrect data type for a data frame \nfeature_cols = [col for col in train.columns if col not in ['target', 'first_active_month', 'card_id']]\nX = train[feature_cols]\n\n# impute missing values\nX = my_imputer.fit_transform(X)\n\n# get the target vector\ny = train['target']","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:52:02.149732Z","iopub.execute_input":"2022-02-25T00:52:02.150003Z","iopub.status.idle":"2022-02-25T00:52:02.770908Z","shell.execute_reply.started":"2022-02-25T00:52:02.149973Z","shell.execute_reply":"2022-02-25T00:52:02.769997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Split test and training set from train dataframe","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.05, random_state=6)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:52:02.772135Z","iopub.execute_input":"2022-02-25T00:52:02.772398Z","iopub.status.idle":"2022-02-25T00:52:03.002479Z","shell.execute_reply.started":"2022-02-25T00:52:02.772369Z","shell.execute_reply":"2022-02-25T00:52:03.001324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Training on any regression models\n**\n\nImporting training models\n\n","metadata":{}},{"cell_type":"code","source":"from sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.ensemble import RandomForestRegressor\n\nreg_predictions = []","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:52:03.006117Z","iopub.execute_input":"2022-02-25T00:52:03.006359Z","iopub.status.idle":"2022-02-25T00:52:03.065673Z","shell.execute_reply.started":"2022-02-25T00:52:03.006332Z","shell.execute_reply":"2022-02-25T00:52:03.064734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train using KNeighborsRegressor\n\n","metadata":{}},{"cell_type":"code","source":"myKNeighborsReg = KNeighborsRegressor(n_neighbors = 3)\n\nmyKNeighborsReg.fit(X_train, y_train)\n\ny_predict_myKNeighborsReg = myKNeighborsReg.predict(X_test)\n\nreg_predictions.append(y_predict_myKNeighborsReg)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:52:03.066855Z","iopub.execute_input":"2022-02-25T00:52:03.067115Z","iopub.status.idle":"2022-02-25T00:52:10.412507Z","shell.execute_reply.started":"2022-02-25T00:52:03.067085Z","shell.execute_reply":"2022-02-25T00:52:10.411393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train using DecisionTreeRegressor\n\n","metadata":{}},{"cell_type":"code","source":"myDecisionTreeReg = DecisionTreeRegressor(random_state = 5)\n\nmyDecisionTreeReg.fit(X_train, y_train)\n\ny_predict_myDecisionTreeReg = myDecisionTreeReg.predict(X_test)\n\nreg_predictions.append(y_predict_myDecisionTreeReg)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:52:10.416219Z","iopub.execute_input":"2022-02-25T00:52:10.416460Z","iopub.status.idle":"2022-02-25T00:52:35.398929Z","shell.execute_reply.started":"2022-02-25T00:52:10.416433Z","shell.execute_reply":"2022-02-25T00:52:35.398094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train using LinearRegression","metadata":{}},{"cell_type":"code","source":"myLinearReg = LinearRegression()\n\nmyLinearReg.fit(X_train, y_train)\n\ny_predict_myLinearReg = myLinearReg.predict(X_test)\n\nreg_predictions.append(y_predict_myLinearReg)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:52:35.400278Z","iopub.execute_input":"2022-02-25T00:52:35.400533Z","iopub.status.idle":"2022-02-25T00:52:36.446773Z","shell.execute_reply.started":"2022-02-25T00:52:35.400505Z","shell.execute_reply":"2022-02-25T00:52:36.445763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train using RandomForestRegressor","metadata":{}},{"cell_type":"code","source":"myRandomForestReg = RandomForestRegressor(n_estimators = 9, bootstrap = True, random_state = 3)\n\nmyRandomForestReg.fit(X_train, y_train)\n\ny_predict_myRandomForestReg = myRandomForestReg.predict(X_test)\n\nreg_predictions.append(y_predict_myRandomForestReg)\n\nprint(X.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:52:36.448498Z","iopub.execute_input":"2022-02-25T00:52:36.449116Z","iopub.status.idle":"2022-02-25T00:55:00.889360Z","shell.execute_reply.started":"2022-02-25T00:52:36.449065Z","shell.execute_reply":"2022-02-25T00:55:00.888273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Check RMSE**","metadata":{}},{"cell_type":"code","source":"from sklearn import metrics\n\nfor model, y_prediction in zip(['K Nearest Neighbor: ', 'Decision Tree: ', 'Linear Regression: ', 'Random Forest: '], reg_predictions):\n    mse = metrics.mean_squared_error(y_test, y_prediction)\n    rmse = np.sqrt(mse)\n    print(model + str(rmse))","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:55:00.891325Z","iopub.execute_input":"2022-02-25T00:55:00.891725Z","iopub.status.idle":"2022-02-25T00:55:00.904403Z","shell.execute_reply.started":"2022-02-25T00:55:00.891682Z","shell.execute_reply":"2022-02-25T00:55:00.903244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Dimensionality Reduction**","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nn = 45 # (n is the number of components (new features)\n# after dimensionality reduction)\nmy_pca = PCA(n_components = n)\n# (X_Train is feature matrix of training set before DR,\n# X_Train_New is feature matrix of training set after DR):\nX_Train_new = my_pca.fit_transform(X_train)\nX_Test_new = my_pca.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:55:00.906032Z","iopub.execute_input":"2022-02-25T00:55:00.906333Z","iopub.status.idle":"2022-02-25T00:55:04.344767Z","shell.execute_reply.started":"2022-02-25T00:55:00.906282Z","shell.execute_reply":"2022-02-25T00:55:04.343324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_pca.explained_variance_ratio_","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:55:04.346628Z","iopub.execute_input":"2022-02-25T00:55:04.347012Z","iopub.status.idle":"2022-02-25T00:55:04.356350Z","shell.execute_reply.started":"2022-02-25T00:55:04.346965Z","shell.execute_reply":"2022-02-25T00:55:04.355407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_pca.n_components_","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:55:04.357847Z","iopub.execute_input":"2022-02-25T00:55:04.358534Z","iopub.status.idle":"2022-02-25T00:55:04.380902Z","shell.execute_reply.started":"2022-02-25T00:55:04.358485Z","shell.execute_reply":"2022-02-25T00:55:04.379596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:55:04.382692Z","iopub.execute_input":"2022-02-25T00:55:04.383331Z","iopub.status.idle":"2022-02-25T00:55:04.399752Z","shell.execute_reply.started":"2022-02-25T00:55:04.383281Z","shell.execute_reply":"2022-02-25T00:55:04.398334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_Train_new.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:55:04.401586Z","iopub.execute_input":"2022-02-25T00:55:04.402080Z","iopub.status.idle":"2022-02-25T00:55:04.416224Z","shell.execute_reply.started":"2022-02-25T00:55:04.402030Z","shell.execute_reply":"2022-02-25T00:55:04.415035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:55:04.418100Z","iopub.execute_input":"2022-02-25T00:55:04.418609Z","iopub.status.idle":"2022-02-25T00:55:04.432220Z","shell.execute_reply.started":"2022-02-25T00:55:04.418555Z","shell.execute_reply":"2022-02-25T00:55:04.431023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_new.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:55:04.433584Z","iopub.execute_input":"2022-02-25T00:55:04.434082Z","iopub.status.idle":"2022-02-25T00:55:04.506477Z","shell.execute_reply.started":"2022-02-25T00:55:04.434027Z","shell.execute_reply":"2022-02-25T00:55:04.502975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_Test_new.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:55:21.119010Z","iopub.execute_input":"2022-02-25T00:55:21.120285Z","iopub.status.idle":"2022-02-25T00:55:21.129344Z","shell.execute_reply.started":"2022-02-25T00:55:21.120222Z","shell.execute_reply":"2022-02-25T00:55:21.127965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reg_predictions_new = []","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:55:29.822293Z","iopub.execute_input":"2022-02-25T00:55:29.822971Z","iopub.status.idle":"2022-02-25T00:55:29.827944Z","shell.execute_reply.started":"2022-02-25T00:55:29.822925Z","shell.execute_reply":"2022-02-25T00:55:29.826416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train using KNeighborsRegressor\n\n","metadata":{}},{"cell_type":"markdown","source":"myKNeighborsReg = KNeighborsRegressor(n_neighbors = 3)\n\nmyKNeighborsReg.fit(X_Train_new, y_train)\n\ny_predict_myKNeighborsReg = myKNeighborsReg.predict(X_Test_new)\n\nreg_predictions_new.append(y_predict_myKNeighborsReg)","metadata":{"execution":{"iopub.status.busy":"2022-02-09T18:12:09.230669Z","iopub.execute_input":"2022-02-09T18:12:09.231295Z","iopub.status.idle":"2022-02-09T18:12:12.9625Z","shell.execute_reply.started":"2022-02-09T18:12:09.231248Z","shell.execute_reply":"2022-02-09T18:12:12.961278Z"}}},{"cell_type":"markdown","source":"Train using DecisionTreeRegressor","metadata":{"execution":{"iopub.status.busy":"2022-02-09T18:12:12.964189Z","iopub.execute_input":"2022-02-09T18:12:12.964516Z","iopub.status.idle":"2022-02-09T18:12:12.970929Z","shell.execute_reply.started":"2022-02-09T18:12:12.964483Z","shell.execute_reply":"2022-02-09T18:12:12.969891Z"}}},{"cell_type":"code","source":"myDecisionTreeReg = DecisionTreeRegressor(random_state = 5)\n\nmyDecisionTreeReg.fit(X_Train_new, y_train)\n\ny_predict_myDecisionTreeReg = myDecisionTreeReg.predict(X_Test_new)\n\nreg_predictions_new.append(y_predict_myDecisionTreeReg)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:55:29.830224Z","iopub.execute_input":"2022-02-25T00:55:29.831103Z","iopub.status.idle":"2022-02-25T00:56:01.939020Z","shell.execute_reply.started":"2022-02-25T00:55:29.831051Z","shell.execute_reply":"2022-02-25T00:56:01.937820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train using LinearRegression","metadata":{}},{"cell_type":"code","source":"myLinearReg = LinearRegression()\n\nmyLinearReg.fit(X_Train_new, y_train)\n\ny_predict_myLinearReg = myLinearReg.predict(X_Test_new)\n\nreg_predictions_new.append(y_predict_myLinearReg)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:56:01.940574Z","iopub.execute_input":"2022-02-25T00:56:01.941168Z","iopub.status.idle":"2022-02-25T00:56:02.355511Z","shell.execute_reply.started":"2022-02-25T00:56:01.941118Z","shell.execute_reply":"2022-02-25T00:56:02.354441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train using RandomForestRegressor","metadata":{}},{"cell_type":"code","source":"myRandomForestReg = RandomForestRegressor(n_estimators = 9, bootstrap = True, random_state = 3)\n\nmyRandomForestReg.fit(X_Train_new, y_train)\n\ny_predict_myRandomForestReg = myRandomForestReg.predict(X_Test_new)\n\nreg_predictions_new.append(y_predict_myRandomForestReg)\n\nprint(X.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:56:02.361312Z","iopub.execute_input":"2022-02-25T00:56:02.364437Z","iopub.status.idle":"2022-02-25T00:59:01.044300Z","shell.execute_reply.started":"2022-02-25T00:56:02.364326Z","shell.execute_reply":"2022-02-25T00:59:01.043287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for model, y_prediction in zip(['K Nearest Neighbor: ', 'Decision Tree: ', 'Linear Regression: ', 'Random Forest: '], reg_predictions_new):\n    mse = metrics.mean_squared_error(y_test, y_prediction)\n    rmse = np.sqrt(mse)\n    print(model + str(rmse))","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:59:01.047898Z","iopub.execute_input":"2022-02-25T00:59:01.048159Z","iopub.status.idle":"2022-02-25T00:59:01.058664Z","shell.execute_reply.started":"2022-02-25T00:59:01.048129Z","shell.execute_reply":"2022-02-25T00:59:01.057680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Repeating above same steps on actual test dataframe**","metadata":{}},{"cell_type":"markdown","source":"Training and Testing on all new features\n\n","metadata":{}},{"cell_type":"code","source":"test_feature_cols = [col for col in test.columns if col not in ['target', 'first_active_month', 'card_id']]\nfinal_test = test[feature_cols]\nfinal_test = my_imputer.fit_transform(final_test)\n\nreg_predictions_final = {}","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:59:01.060096Z","iopub.execute_input":"2022-02-25T00:59:01.060562Z","iopub.status.idle":"2022-02-25T00:59:01.511887Z","shell.execute_reply.started":"2022-02-25T00:59:01.060497Z","shell.execute_reply":"2022-02-25T00:59:01.510883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train using KNeighborsRegressor\n\n","metadata":{}},{"cell_type":"code","source":"myKNeighborsReg = KNeighborsRegressor(n_neighbors = 3)\n\nmyKNeighborsReg.fit(X, y)\n\ny_predict_myKNeighborsReg = myKNeighborsReg.predict(final_test)\n\nreg_predictions_final['K Nearest Neighbor: '] = y_predict_myKNeighborsReg","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:59:01.514802Z","iopub.execute_input":"2022-02-25T00:59:01.515178Z","iopub.status.idle":"2022-02-25T00:59:23.142952Z","shell.execute_reply.started":"2022-02-25T00:59:01.515135Z","shell.execute_reply":"2022-02-25T00:59:23.142080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train using DecisionTreeRegressor\n\n","metadata":{}},{"cell_type":"code","source":"myDecisionTreeReg = DecisionTreeRegressor(random_state = 5)\n\nmyDecisionTreeReg.fit(X, y)\n\ny_predict_myDecisionTreeReg = myDecisionTreeReg.predict(final_test)\n\nreg_predictions_final['Decision Tree: ']= y_predict_myDecisionTreeReg","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:59:23.144121Z","iopub.execute_input":"2022-02-25T00:59:23.144342Z","iopub.status.idle":"2022-02-25T00:59:49.743614Z","shell.execute_reply.started":"2022-02-25T00:59:23.144305Z","shell.execute_reply":"2022-02-25T00:59:49.742725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train using LinearRegression","metadata":{}},{"cell_type":"code","source":"myLinearReg = LinearRegression()\n\nmyLinearReg.fit(X, y)\n\ny_predict_myLinearReg = myLinearReg.predict(final_test)\n\nreg_predictions_final['Linear Regression: '] = y_predict_myLinearReg","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:59:49.745297Z","iopub.execute_input":"2022-02-25T00:59:49.745556Z","iopub.status.idle":"2022-02-25T00:59:50.751607Z","shell.execute_reply.started":"2022-02-25T00:59:49.745526Z","shell.execute_reply":"2022-02-25T00:59:50.750555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train using RandomForestRegressor","metadata":{}},{"cell_type":"code","source":"myRandomForestReg = RandomForestRegressor(n_estimators = 9, bootstrap = True, random_state = 3)\n\nmyRandomForestReg.fit(X, y)\n\ny_predict_myRandomForestReg = myRandomForestReg.predict(final_test)\n\nreg_predictions_final['Random Forest: '] = y_predict_myRandomForestReg\n\nprint(X.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T00:59:50.757345Z","iopub.execute_input":"2022-02-25T00:59:50.757940Z","iopub.status.idle":"2022-02-25T01:02:20.741064Z","shell.execute_reply.started":"2022-02-25T00:59:50.757880Z","shell.execute_reply":"2022-02-25T01:02:20.740062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for model, y_prediction in reg_predictions_final.items():\n    mse = metrics.mean_squared_error(target.iloc[:len(y_prediction)], y_prediction)\n    rmse = np.sqrt(mse)\n    print(model + str(rmse))","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:02:20.742571Z","iopub.execute_input":"2022-02-25T01:02:20.742901Z","iopub.status.idle":"2022-02-25T01:02:20.762221Z","shell.execute_reply.started":"2022-02-25T01:02:20.742860Z","shell.execute_reply":"2022-02-25T01:02:20.761551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Training and testing using dimensionality reduction\n\n","metadata":{}},{"cell_type":"code","source":"reg_predictions_final_dr = {}\n\nn = 45\n\nmy_pca = PCA(n_components = n)\n\nX_new = my_pca.fit_transform(X)\nfinal_test_new = my_pca.transform(final_test)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:02:20.763647Z","iopub.execute_input":"2022-02-25T01:02:20.764008Z","iopub.status.idle":"2022-02-25T01:02:24.376958Z","shell.execute_reply.started":"2022-02-25T01:02:20.763965Z","shell.execute_reply":"2022-02-25T01:02:24.375756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train using KNeighborsRegressor","metadata":{}},{"cell_type":"code","source":"myKNeighborsReg = KNeighborsRegressor(n_neighbors = 3)\n\nmyKNeighborsReg.fit(X_new, y)\n\ny_predict_myKNeighborsReg = myKNeighborsReg.predict(final_test_new)\n\nreg_predictions_final_dr['K Nearest Neighbor: '] = y_predict_myKNeighborsReg","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:02:24.378492Z","iopub.execute_input":"2022-02-25T01:02:24.378973Z","iopub.status.idle":"2022-02-25T01:02:34.741389Z","shell.execute_reply.started":"2022-02-25T01:02:24.378929Z","shell.execute_reply":"2022-02-25T01:02:34.740506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train using DecisionTreeRegressor\n\n","metadata":{}},{"cell_type":"code","source":"myDecisionTreeReg = DecisionTreeRegressor(random_state = 5)\n\nmyDecisionTreeReg.fit(X_new, y)\n\ny_predict_myDecisionTreeReg = myDecisionTreeReg.predict(final_test_new)\n\nreg_predictions_final_dr['Decision Tree: ']= y_predict_myDecisionTreeReg","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:02:34.744933Z","iopub.execute_input":"2022-02-25T01:02:34.745193Z","iopub.status.idle":"2022-02-25T01:03:08.244941Z","shell.execute_reply.started":"2022-02-25T01:02:34.745152Z","shell.execute_reply":"2022-02-25T01:03:08.243925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train using LinearRegression\n\n","metadata":{}},{"cell_type":"code","source":"myLinearReg = LinearRegression()\n\nmyLinearReg.fit(X_new, y)\n\ny_predict_myLinearReg = myLinearReg.predict(final_test_new)\n\nreg_predictions_final_dr['Linear Regression: '] = y_predict_myLinearReg","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:03:08.246185Z","iopub.execute_input":"2022-02-25T01:03:08.246444Z","iopub.status.idle":"2022-02-25T01:03:08.666581Z","shell.execute_reply.started":"2022-02-25T01:03:08.246414Z","shell.execute_reply":"2022-02-25T01:03:08.665526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train using RandomForestRegressor\n\n","metadata":{}},{"cell_type":"code","source":"myRandomForestReg = RandomForestRegressor(n_estimators = 9, bootstrap = True, random_state = 3)\n\nmyRandomForestReg.fit(X_new, y)\n\ny_predict_myRandomForestReg = myRandomForestReg.predict(final_test_new)\n\nreg_predictions_final_dr['Random Forest: '] = y_predict_myRandomForestReg\n\nprint(X.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:03:08.668366Z","iopub.execute_input":"2022-02-25T01:03:08.668945Z","iopub.status.idle":"2022-02-25T01:06:16.468733Z","shell.execute_reply.started":"2022-02-25T01:03:08.668895Z","shell.execute_reply":"2022-02-25T01:06:16.467759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for model, y_prediction in reg_predictions_final_dr.items():\n    mse = metrics.mean_squared_error(target.iloc[:len(y_prediction)], y_prediction)\n    rmse = np.sqrt(mse)\n    print(model + str(rmse))","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:06:16.470189Z","iopub.execute_input":"2022-02-25T01:06:16.470462Z","iopub.status.idle":"2022-02-25T01:06:16.487673Z","shell.execute_reply.started":"2022-02-25T01:06:16.470432Z","shell.execute_reply":"2022-02-25T01:06:16.487045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Applying Advanced Models for Boosting prediction **","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport math\nimport datetime\n\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom sklearn.model_selection import KFold\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.linear_model import Ridge, Lasso\nfrom lightgbm.sklearn import LGBMRegressor\nfrom xgboost.sklearn import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin\nfrom mlxtend.regressor import StackingCVRegressor\n\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import cross_val_predict\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.metrics import make_scorer\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom IPython.display import Image\nimport warnings\n\n%matplotlib inline\nwarnings.filterwarnings('ignore')\nsns.set(style='white', context='notebook', palette='deep')","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:14:57.077024Z","iopub.execute_input":"2022-02-25T01:14:57.077333Z","iopub.status.idle":"2022-02-25T01:14:57.856485Z","shell.execute_reply.started":"2022-02-25T01:14:57.077301Z","shell.execute_reply":"2022-02-25T01:14:57.855600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = reduce_mem_usage(pd.read_csv(\"../input/elo-merchant-category-recommendation/train.csv\"))\ndf_test = reduce_mem_usage(pd.read_csv(\"../input/elo-merchant-category-recommendation/test.csv\"))\ndf_historical =reduce_mem_usage(pd.read_csv(\"../input/elo-merchant-category-recommendation/historical_transactions.csv\",parse_dates=['purchase_date']))\ndf_new =reduce_mem_usage(pd.read_csv(\"../input/elo-merchant-category-recommendation/new_merchant_transactions.csv\",parse_dates=['purchase_date']))","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:14:57.858129Z","iopub.execute_input":"2022-02-25T01:14:57.858391Z","iopub.status.idle":"2022-02-25T01:16:40.539106Z","shell.execute_reply.started":"2022-02-25T01:14:57.858359Z","shell.execute_reply":"2022-02-25T01:16:40.538049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = reduce_mem_usage(pd.read_csv(\"../input/elo-merchant-category-recommendation/train.csv\",parse_dates=['first_active_month']))\ndf_test = reduce_mem_usage(pd.read_csv(\"../input/elo-merchant-category-recommendation/test.csv\",parse_dates=['first_active_month']))","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:16:40.540653Z","iopub.execute_input":"2022-02-25T01:16:40.540989Z","iopub.status.idle":"2022-02-25T01:16:41.027864Z","shell.execute_reply.started":"2022-02-25T01:16:40.540944Z","shell.execute_reply":"2022-02-25T01:16:41.026806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Historical and new transactions data\nhist_trans = df_historical\nnew_trans = df_new\n\n# Train and Test data\ntrain = df_train\ntest = df_test\n\ntrain_idx = train.shape[0]\ntest_idx = test.shape[0]\n\nprint(\"--------------------------\")\nprint(\"Train shape: \", train.shape)\nprint(\"Test shape: \", test.shape)\nprint(\"--------------------------\")\nprint(\"Historical transactions shape: \", hist_trans.shape)\nprint(\"New transactions shape: \", new_trans.shape)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:16:41.029772Z","iopub.execute_input":"2022-02-25T01:16:41.029985Z","iopub.status.idle":"2022-02-25T01:16:41.038573Z","shell.execute_reply.started":"2022-02-25T01:16:41.029958Z","shell.execute_reply":"2022-02-25T01:16:41.037782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Description","metadata":{}},{"cell_type":"code","source":"print(\"----------------------------------------------------------------\")\nprint(\"Train\")\nprint(\"----------------------------------------------------------------\")\nprint(train.info())\nprint(\"\\n----------------------------------------------------------------\")\nprint(\"Test\")\nprint(\"----------------------------------------------------------------\")\nprint(train.info())\nprint(\"\\n----------------------------------------------------------------\")\nprint(\"Historical transactions\")\nprint(\"----------------------------------------------------------------\")\nprint(hist_trans.info())\nprint(\"\\n----------------------------------------------------------------\")\nprint(\"New transactions\")\nprint(\"----------------------------------------------------------------\")\nprint(new_trans.info())","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:16:41.039697Z","iopub.execute_input":"2022-02-25T01:16:41.039968Z","iopub.status.idle":"2022-02-25T01:16:41.137279Z","shell.execute_reply.started":"2022-02-25T01:16:41.039925Z","shell.execute_reply":"2022-02-25T01:16:41.136324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:16:41.138632Z","iopub.execute_input":"2022-02-25T01:16:41.138880Z","iopub.status.idle":"2022-02-25T01:16:41.157852Z","shell.execute_reply.started":"2022-02-25T01:16:41.138849Z","shell.execute_reply":"2022-02-25T01:16:41.156947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()\n","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:16:41.159392Z","iopub.execute_input":"2022-02-25T01:16:41.159607Z","iopub.status.idle":"2022-02-25T01:16:41.172538Z","shell.execute_reply.started":"2022-02-25T01:16:41.159578Z","shell.execute_reply":"2022-02-25T01:16:41.171323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Target column¶\n","metadata":{}},{"cell_type":"code","source":"print(\"Target description:\\n\\n\", train['target'].describe())\nprint(\"\\n--------------------------------------------------------------------------------------------\")\nprint(\"\\nTarget values:\\n\\n\", train['target'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:16:41.174820Z","iopub.execute_input":"2022-02-25T01:16:41.175062Z","iopub.status.idle":"2022-02-25T01:16:41.232228Z","shell.execute_reply.started":"2022-02-25T01:16:41.175032Z","shell.execute_reply":"2022-02-25T01:16:41.231101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(20,6))\nax1, ax2 = axes.flatten()\n\n# Distribution\nsns.distplot(train['target'], ax=ax1, color='Green')\n\n# Sorted correlations with target\nsorted_corrs = train.corr()['target'].sort_values(ascending=False)\nsns.heatmap(train[sorted_corrs.index].corr(), ax=ax2)\n\nax1.set_title('Target Distribution')\nax2.set_title('Correlations')\nplt.show()\ndel sorted_corrs","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:16:41.234133Z","iopub.execute_input":"2022-02-25T01:16:41.234728Z","iopub.status.idle":"2022-02-25T01:16:42.857387Z","shell.execute_reply.started":"2022-02-25T01:16:41.234680Z","shell.execute_reply":"2022-02-25T01:16:42.856389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There seem to be 2207 values around -33 for the target column, which follows a normal distribution. Let's take that into account. Also, 'feature_3' correlates better than 'feature_2' and 'feature_1' with 'target'.\n\nLet's confirm the number of values under -30.","metadata":{}},{"cell_type":"code","source":"under_30 = train.loc[train['target'] < -30, 'target'].count()\nprint(\"Under -30:\", under_30, \"values.\")","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:16:42.860383Z","iopub.execute_input":"2022-02-25T01:16:42.860613Z","iopub.status.idle":"2022-02-25T01:16:42.869381Z","shell.execute_reply.started":"2022-02-25T01:16:42.860583Z","shell.execute_reply":"2022-02-25T01:16:42.868677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cleaning","metadata":{}},{"cell_type":"markdown","source":"**Missing**","metadata":{}},{"cell_type":"code","source":"print(\"MISSING VALUES BEFORE CLEANING\\n\")\nprint(\"--------------------------------------------------\\nTrain:\\n--------------------------------------------------\\n\", train.isnull().sum())\nprint(\"\\n--------------------------------------------------\\nTest:\\n--------------------------------------------------\\n\", test.isnull().sum())\nprint(\"\\n--------------------------------------------------\\nHistorical transactions:\\n--------------------------------------------------\\n\", hist_trans.isnull().sum())\nprint(\"\\n--------------------------------------------------\\nNew transactions:\\n--------------------------------------------------\\n\", new_trans.isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:16:42.870578Z","iopub.execute_input":"2022-02-25T01:16:42.871048Z","iopub.status.idle":"2022-02-25T01:16:57.340239Z","shell.execute_reply.started":"2022-02-25T01:16:42.871013Z","shell.execute_reply":"2022-02-25T01:16:57.339266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train/Test¶","metadata":{}},{"cell_type":"markdown","source":"There's no null values for train. However, there seem to be one observation with a missing 'first_active_month' in test.\n\n","metadata":{}},{"cell_type":"code","source":"test_missing = test[test.isnull()['first_active_month']]\nidx_test_missing = test_missing.index\ntest_missing","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:16:57.341658Z","iopub.execute_input":"2022-02-25T01:16:57.342074Z","iopub.status.idle":"2022-02-25T01:16:57.369323Z","shell.execute_reply.started":"2022-02-25T01:16:57.342042Z","shell.execute_reply":"2022-02-25T01:16:57.368053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can look for all the observations that match the same 'feature_1', 'feature_2' and 'feature_3' values as that, and replace it with the 'first_active_month' that corresponds to their mode.","metadata":{}},{"cell_type":"code","source":"same_category = test[(test['feature_1'] == 5) & (test['feature_2'] == 2) & (test['feature_3'] == 1)]\ntest.loc[idx_test_missing, 'first_active_month'] = same_category['first_active_month'].mode()[0]\n\ndel same_category\ntest.iloc[11578]","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:16:57.371190Z","iopub.execute_input":"2022-02-25T01:16:57.372278Z","iopub.status.idle":"2022-02-25T01:16:57.391333Z","shell.execute_reply.started":"2022-02-25T01:16:57.372230Z","shell.execute_reply":"2022-02-25T01:16:57.390301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Historical transactions¶\n","metadata":{}},{"cell_type":"markdown","source":"Similarly, in this case let's drop the missing rows for 'category_3' and 'merchant_id' (less than 1% of total).\n\n","metadata":{}},{"cell_type":"code","source":"hist_trans.dropna(subset=['category_3', 'merchant_id'], inplace=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:16:57.392941Z","iopub.execute_input":"2022-02-25T01:16:57.393316Z","iopub.status.idle":"2022-02-25T01:17:06.175987Z","shell.execute_reply.started":"2022-02-25T01:16:57.393270Z","shell.execute_reply":"2022-02-25T01:17:06.174704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For 'category_2', since it has about 10% of missing values, let's replace them with the rounded average value (since values for this column include [1.0, 2.0, 3.0, 4.0, 5.0]), as seen below.","metadata":{}},{"cell_type":"code","source":"hist_trans['category_2'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:17:06.177556Z","iopub.execute_input":"2022-02-25T01:17:06.177897Z","iopub.status.idle":"2022-02-25T01:17:10.218383Z","shell.execute_reply.started":"2022-02-25T01:17:06.177850Z","shell.execute_reply":"2022-02-25T01:17:10.217608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(hist_trans[hist_trans['category_2'].isnull()])","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:17:10.219585Z","iopub.execute_input":"2022-02-25T01:17:10.219843Z","iopub.status.idle":"2022-02-25T01:17:10.727017Z","shell.execute_reply.started":"2022-02-25T01:17:10.219813Z","shell.execute_reply":"2022-02-25T01:17:10.726165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist_trans['category_2'] = pd.to_numeric(hist_trans['category_2'], errors='coerce')","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:17:10.728287Z","iopub.execute_input":"2022-02-25T01:17:10.728505Z","iopub.status.idle":"2022-02-25T01:17:10.749487Z","shell.execute_reply.started":"2022-02-25T01:17:10.728476Z","shell.execute_reply":"2022-02-25T01:17:10.748544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist_trans = hist_trans.dropna(subset=['category_2'])","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:17:10.750761Z","iopub.execute_input":"2022-02-25T01:17:10.750973Z","iopub.status.idle":"2022-02-25T01:17:13.145208Z","shell.execute_reply.started":"2022-02-25T01:17:10.750945Z","shell.execute_reply":"2022-02-25T01:17:13.144356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist_trans['category_2'] = hist_trans['category_2'].astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:17:13.146784Z","iopub.execute_input":"2022-02-25T01:17:13.147018Z","iopub.status.idle":"2022-02-25T01:17:13.353626Z","shell.execute_reply.started":"2022-02-25T01:17:13.146989Z","shell.execute_reply":"2022-02-25T01:17:13.352742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist_trans['category_2'].fillna((math.floor(hist_trans['category_2'].mean())), inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:17:13.354998Z","iopub.execute_input":"2022-02-25T01:17:13.355280Z","iopub.status.idle":"2022-02-25T01:17:13.402091Z","shell.execute_reply.started":"2022-02-25T01:17:13.355251Z","shell.execute_reply":"2022-02-25T01:17:13.401217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# New transactions¶\nOnce more, since there are missing values in 'category_3', 'merchant_id', 'category_2' and they add up to no more than about 5%, let's drop the corresponding rows.","metadata":{}},{"cell_type":"code","source":"new_trans.dropna(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:17:13.403475Z","iopub.execute_input":"2022-02-25T01:17:13.403699Z","iopub.status.idle":"2022-02-25T01:17:14.524792Z","shell.execute_reply.started":"2022-02-25T01:17:13.403671Z","shell.execute_reply":"2022-02-25T01:17:14.523971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lastly, let's confirm no null values are present after cleaning.\n","metadata":{}},{"cell_type":"code","source":"print(\"MISSING VALUES AFTER CLEANING\\n\")\nprint(\"--------------------------------------------------\\nTrain:\\n--------------------------------------------------\\n\", train.isnull().sum())\nprint(\"\\n--------------------------------------------------\\nTest:\\n--------------------------------------------------\\n\", test.isnull().sum())\n#print(\"\\n--------------------------------------------------\\nMerchant:\\n--------------------------------------------------\\n\", merchants.isnull().sum())\nprint(\"\\n--------------------------------------------------\\nHistorical transactions:\\n--------------------------------------------------\\n\", hist_trans.isnull().sum())\nprint(\"\\n--------------------------------------------------\\nNew transactions:\\n--------------------------------------------------\\n\", new_trans.isnull().sum())","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:17:14.526339Z","iopub.execute_input":"2022-02-25T01:17:14.526571Z","iopub.status.idle":"2022-02-25T01:17:27.432057Z","shell.execute_reply.started":"2022-02-25T01:17:14.526541Z","shell.execute_reply":"2022-02-25T01:17:27.431165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature engineering","metadata":{}},{"cell_type":"code","source":"# Merge train and test for data processing\ndata = pd.concat([train, test], ignore_index=True)\n\n# Check shapes match\nprint(\"Train ({}) + Test ({}) observations: {}\".format(train.shape[0], test.shape[0], train.shape[0] + test.shape[0]))\nprint(\"Merged shape:\", data.shape)\n\ndel train\ndel test","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:17:27.433331Z","iopub.execute_input":"2022-02-25T01:17:27.433567Z","iopub.status.idle":"2022-02-25T01:17:27.450258Z","shell.execute_reply.started":"2022-02-25T01:17:27.433538Z","shell.execute_reply":"2022-02-25T01:17:27.448910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train/Test¶","metadata":{}},{"cell_type":"code","source":"# Year and month, separately\ndata['year'] = data['first_active_month'].dt.year\ndata['month'] = data['first_active_month'].dt.month\n\n# Elapsed time, until the latest date on the dataset\ndata['elapsed_time'] = (datetime.date(2018, 2, 1) - data['first_active_month'].dt.date).dt.days\n\n# Categorical features: 'feature_1', 'feature_2' and 'feature_3'\ncont = 1\nfor col in ['feature_1', 'feature_2', 'feature_3']:\n    dummy_col = pd.get_dummies(data[col], prefix='f{}'.format(cont))\n    data = pd.concat([data, dummy_col], axis=1)\n    data.drop(col, axis=1, inplace=True)\n    cont += 1\n    \ndata.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:17:27.453371Z","iopub.execute_input":"2022-02-25T01:17:27.453624Z","iopub.status.idle":"2022-02-25T01:17:29.083134Z","shell.execute_reply.started":"2022-02-25T01:17:27.453584Z","shell.execute_reply":"2022-02-25T01:17:29.082185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Historical + New transactions\nLet's create a column called 'new' on 'hist_trans' and 'new_trans' such that, before concatening them, they have the age reference:\n\n* 1: New\n* 0: Historical","metadata":{}},{"cell_type":"code","source":"new_trans['new'] = 1\nhist_trans['new'] = 0\n\n# Concatenate new_trans and hist_trans\ntrans_data = pd.concat([new_trans, hist_trans])\n\ndel new_trans\ndel hist_trans","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:17:29.084632Z","iopub.execute_input":"2022-02-25T01:17:29.084851Z","iopub.status.idle":"2022-02-25T01:17:34.499538Z","shell.execute_reply.started":"2022-02-25T01:17:29.084822Z","shell.execute_reply":"2022-02-25T01:17:34.498611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"More preprocessing: 'category_1', 'category_2' and 'category_3'.\n\n","metadata":{}},{"cell_type":"code","source":"# Change Yes/No for 0/1 in 'authorized_flag' and 'category_1'\nyes_no_dict = {'Y':1, 'N':0}\ntrans_data['authorized_flag'] = trans_data['authorized_flag'].map(yes_no_dict)\ntrans_data['category_1'] = trans_data['category_1'].map(yes_no_dict)\n\n# Create five different cols for 'category_2'\ndummy_col = pd.get_dummies(trans_data['category_2'], prefix='category_2')\ntrans_data = pd.concat([trans_data, dummy_col], axis=1)\ntrans_data.drop('category_2', axis=1, inplace=True)\n    \n# Create three different cols for categorical A/B/C in 'category_3'\ndummy_col = pd.get_dummies(trans_data['category_3'], prefix='cat3')\ntrans_data = pd.concat([trans_data, dummy_col], axis=1)\ntrans_data.drop('category_3', axis=1, inplace=True)\n\ntrans_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:17:34.501297Z","iopub.execute_input":"2022-02-25T01:17:34.501569Z","iopub.status.idle":"2022-02-25T01:17:59.565406Z","shell.execute_reply.started":"2022-02-25T01:17:34.501537Z","shell.execute_reply":"2022-02-25T01:17:59.564413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Aggregate function¶\nAggregate function, grouped by 'card_id': min, max, mean, median, std, sum, nunique, range.\n\nAdded:\n* Count on 'installments' and 'purchase_amount'.\n* Mode() on 'new' column (previously created).\n* Mean on new trans_data's category_2 dummy columns.\n* Mean on trans_data's category_4.\n* Mean on 'cat3_A', 'cat3_B' and 'cat3_C' (old 'category_3').\n* Mean on merchants' new dummy columns.","metadata":{}},{"cell_type":"code","source":"def aggregate_historical_transactions(trans_data):\n    \n    trans_data.loc[:, 'purchase_date'] = pd.DatetimeIndex(trans_data['purchase_date']).astype(np.int64)*1e-9\n    \n    agg_func = {\n        'authorized_flag': ['sum', 'mean'],\n        'category_1': ['mean'],\n        'category_2_1.0': ['mean'],\n        'category_2_2.0': ['mean'],\n        'category_2_3.0': ['mean'],\n        'category_2_4.0': ['mean'],\n        'category_2_5.0': ['mean'],\n        'cat3_A': ['mean'],\n        'cat3_B': ['mean'],\n        'cat3_C': ['mean'],\n        'merchant_id': ['nunique'],\n        'merchant_category_id': ['nunique'],\n        'state_id': ['nunique'],\n        'city_id': ['nunique'],\n        'subsector_id': ['nunique'],\n        'purchase_amount': ['count', 'sum', 'median', 'max', 'min', 'std'],\n        'installments': ['count', 'sum', 'median', 'max', 'min', 'std'],\n        'purchase_date': [np.ptp],\n        'month_lag': ['min', 'max'],\n        'new':[lambda x:x.value_counts().index[0]] # Mode\n        }\n    \n    agg_history = trans_data.groupby(['card_id']).agg(agg_func)\n    agg_history.columns = ['_'.join(col).strip() for col in agg_history.columns.values]\n    agg_history.reset_index(inplace=True)\n\n    df = (trans_data.groupby('card_id').size().reset_index(name='transactions_count'))\n    \n    agg_history = pd.merge(df, agg_history, on='card_id', how='left')\n    \n    return agg_history\n\ntrans_data = aggregate_historical_transactions(trans_data)\ntrans_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:17:59.567392Z","iopub.execute_input":"2022-02-25T01:17:59.567731Z","iopub.status.idle":"2022-02-25T01:21:32.664973Z","shell.execute_reply.started":"2022-02-25T01:17:59.567686Z","shell.execute_reply":"2022-02-25T01:21:32.664047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data preparation\n\n# Merge","metadata":{}},{"cell_type":"code","source":"# Merge data (train + test) with trans_data (historical + new transactions)\nprocessed_data = pd.merge(data, trans_data, on='card_id', how='left')\ndel data\ndel trans_data\nprint(processed_data.shape)\nprocessed_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:21:32.669492Z","iopub.execute_input":"2022-02-25T01:21:32.669738Z","iopub.status.idle":"2022-02-25T01:21:33.421133Z","shell.execute_reply.started":"2022-02-25T01:21:32.669709Z","shell.execute_reply":"2022-02-25T01:21:33.420168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Final Train/Test¶\n","metadata":{}},{"cell_type":"code","source":"# Train and Test\ntrain = processed_data[:train_idx]\ntest = processed_data[train_idx:]\n\ndel processed_data\n\n# There are some nan values after feature eng in 'purchase_amount_std' and 'installments_std'\ncols = ['purchase_amount_std', 'installments_std']\n\nfor col in cols:\n    train[col].fillna((train[col].value_counts().index[0]), inplace=True)\n    test[col].fillna((test[col].value_counts().index[0]), inplace=True)\n\ntarget = train['target']\n\ncols_2_remove = ['target', 'card_id', 'first_active_month']\nfor col in cols_2_remove:  \n    del train[col]\n    del test[col] \n\n# Check on shapes\nprint(\"--------------------------\")\nprint(\"Train shape: \", train.shape)\nprint(\"Test shape: \", test.shape)\nprint(\"--------------------------\")","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:21:33.422895Z","iopub.execute_input":"2022-02-25T01:21:33.423257Z","iopub.status.idle":"2022-02-25T01:21:33.504928Z","shell.execute_reply.started":"2022-02-25T01:21:33.423213Z","shell.execute_reply":"2022-02-25T01:21:33.503975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling and Testing","metadata":{}},{"cell_type":"markdown","source":"# ****LightGBM****","metadata":{}},{"cell_type":"code","source":"lgb_params = {'num_leaves': 50,\n         'min_data_in_leaf': 30, \n         'objective':'regression',\n         'max_depth': -1,\n         'learning_rate': 0.005,\n         \"boosting\": \"gbdt\",\n         \"feature_fraction\": 0.9,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.9,\n         \"bagging_seed\": 11,\n         \"metric\": 'rmse',\n         \"lambda_l1\": 0.1,\n         \"verbosity\": -1}\n\nFOLDs = KFold(n_splits=5, shuffle=True, random_state=1989)\n\noof_lgb = np.zeros(len(train))\npredictions_lgb = np.zeros(len(test))\n\nfeatures_lgb = list(train.columns)\nfeature_importance_df_lgb = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(FOLDs.split(train)):\n    trn_data = lgb.Dataset(train.iloc[trn_idx], label=target.iloc[trn_idx])\n    val_data = lgb.Dataset(train.iloc[val_idx], label=target.iloc[val_idx])\n\n    print(\"LGB \" + str(fold_) + \"-\" * 50)\n    num_round = 2000\n    clf = lgb.train(lgb_params, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=200, early_stopping_rounds = 2000)\n    oof_lgb[val_idx] = clf.predict(train.iloc[val_idx], num_iteration=clf.best_iteration)\n\n    fold_importance_df_lgb = pd.DataFrame()\n    fold_importance_df_lgb[\"feature\"] = features_lgb\n    fold_importance_df_lgb[\"importance\"] = clf.feature_importance()\n    fold_importance_df_lgb[\"fold\"] = fold_ + 1\n    feature_importance_df_lgb = pd.concat([feature_importance_df_lgb, fold_importance_df_lgb], axis=0)\n    predictions_lgb += clf.predict(test, num_iteration=clf.best_iteration) / FOLDs.n_splits\n    \n\ndel fold_importance_df_lgb\ndel trn_data\ndel val_data\n\nprint(np.sqrt(mean_squared_error(oof_lgb, target)))","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:21:33.506233Z","iopub.execute_input":"2022-02-25T01:21:33.506535Z","iopub.status.idle":"2022-02-25T01:25:58.403503Z","shell.execute_reply.started":"2022-02-25T01:21:33.506497Z","shell.execute_reply":"2022-02-25T01:25:58.402426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Xgboost","metadata":{}},{"cell_type":"code","source":"import xgboost as xgb","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:25:58.404924Z","iopub.execute_input":"2022-02-25T01:25:58.405217Z","iopub.status.idle":"2022-02-25T01:25:58.409823Z","shell.execute_reply.started":"2022-02-25T01:25:58.405184Z","shell.execute_reply":"2022-02-25T01:25:58.408662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.rename(index=str, columns={\"new_<lambda>\": \"new_mode\"}, inplace=True)\ntest.rename(index=str, columns={\"new_<lambda>\": \"new_mode\"}, inplace=True)\n\nxgb_params = { 'max_depth': 7, 'subsample': 0.8, 'colsample_bytree': 0.8, \n          'objective': 'reg:linear', 'eval_metric': 'rmse', 'silent': True, 'eta':1}\n\nFOLDs = KFold(n_splits=5, shuffle=True, random_state=1989)\n\noof_xgb = np.zeros(len(train))\npredictions_xgb = np.zeros(len(test))\n\nfor fold_, (trn_idx, val_idx) in enumerate(FOLDs.split(train)):\n    trn_data = xgb.DMatrix(data=train.iloc[trn_idx], label=target.iloc[trn_idx])\n    val_data = xgb.DMatrix(data=train.iloc[val_idx], label=target.iloc[val_idx])\n    watchlist = [(trn_data, 'train'), (val_data, 'valid')]\n    print(\"xgb \" + str(fold_) + \"-\" * 50)\n    num_round = 2000\n    xgb_model = xgb.train(xgb_params, trn_data, num_round, watchlist, early_stopping_rounds=100, verbose_eval=200)\n    oof_xgb[val_idx] = xgb_model.predict(xgb.DMatrix(train.iloc[val_idx]), ntree_limit=xgb_model.best_ntree_limit+50)\n\n    predictions_xgb += xgb_model.predict(xgb.DMatrix(test), ntree_limit=xgb_model.best_ntree_limit+50) / FOLDs.n_splits\n\ndel trn_data\ndel val_data\ndel watchlist\n\nnp.sqrt(mean_squared_error(oof_xgb, target))","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:25:58.411158Z","iopub.execute_input":"2022-02-25T01:25:58.411377Z","iopub.status.idle":"2022-02-25T01:29:50.250209Z","shell.execute_reply.started":"2022-02-25T01:25:58.411348Z","shell.execute_reply":"2022-02-25T01:29:50.249266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Summary of results¶\n","metadata":{}},{"cell_type":"code","source":"print(\"-----------------\\nScores on train\\n-----------------\")\nprint('lgb:', np.sqrt(mean_squared_error(oof_lgb, target)))\nprint('xgb:', np.sqrt(mean_squared_error(oof_xgb, target)))\n\ntotal_sum = 0.5*oof_lgb + 0.5*oof_xgb\n\nprint(\"CV score: {:<8.5f}\".format(mean_squared_error(total_sum, target)**0.5))","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:29:50.251599Z","iopub.execute_input":"2022-02-25T01:29:50.251828Z","iopub.status.idle":"2022-02-25T01:29:50.267280Z","shell.execute_reply.started":"2022-02-25T01:29:50.251798Z","shell.execute_reply":"2022-02-25T01:29:50.266168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature importance¶\n","metadata":{}},{"cell_type":"code","source":"cols = (feature_importance_df_lgb[[\"feature\", \"importance\"]]\n        .groupby(\"feature\")\n        .mean()\n        .sort_values(by=\"importance\", ascending=False)[:1000].index)\n\nbest_features = feature_importance_df_lgb.loc[feature_importance_df_lgb.feature.isin(cols)]\n\nplt.figure(figsize=(14,14))\nsns.barplot(x=\"importance\",\n            y=\"feature\",\n            data=best_features.sort_values(by=\"importance\",\n                                           ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')\ndel feature_importance_df_lgb","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:29:50.268498Z","iopub.execute_input":"2022-02-25T01:29:50.268998Z","iopub.status.idle":"2022-02-25T01:29:53.483271Z","shell.execute_reply.started":"2022-02-25T01:29:50.268965Z","shell.execute_reply":"2022-02-25T01:29:53.482149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensembled model: averaged and stacked¶\nThe follwing tests lgbm, xgb, catboost, random forest, decision tree, knn, ridge and lasso models individual performance, and compared for averaged and stacked models.","metadata":{}},{"cell_type":"markdown","source":"# Model definition\n","metadata":{}},{"cell_type":"code","source":"# Model definition\ntrain_y = target\n\n# Same lgbm and xgb models as before\nlgbm_model = LGBMRegressor(\n                objective=\"regression\", metric=\"rmse\", \n                max_depth=7, min_child_samples=20, \n                reg_alpha= 1, reg_lambda=1,\n                num_leaves=64, learning_rate=0.001, \n                subsample=0.8, colsample_bytree=0.8, \n                verbosity=-1\n)\n\nxgb_model = XGBRegressor(\n                eta=0.001, max_depth=7, \n                subsample=0.8, colsample_bytree=0.8, \n                objective='reg:linear', eval_metric='rmse', \n                silent=True\n)\n\n\n# Test catboost, random forest, decision tree, knn, ridge and lasso models individual performance, for averaged and stacked model\ncatboost_model = CatBoostRegressor(iterations=150)\nrf_model = RandomForestRegressor(n_estimators=25, min_samples_leaf=25, min_samples_split=25)\ntree_model = DecisionTreeRegressor(min_samples_leaf=25, min_samples_split=25)\nknn_model = KNeighborsRegressor(n_neighbors=25, weights='distance')\nridge_model = Ridge(alpha=75.0)\nlasso_model = Lasso(alpha=0.75)\n\n# ------------------------------------------------------------------------------------------------\n# Average regressor\nclass AveragingRegressor(BaseEstimator, RegressorMixin, TransformerMixin):\n    def __init__(self, regressors):\n        self.regressors = regressors\n        self.predictions = None\n\n    def fit(self, X, y):\n        for regr in self.regressors:\n            regr.fit(X, y)\n        return self\n\n    def predict(self, X):\n        self.predictions = np.column_stack([regr.predict(X) for regr in self.regressors])\n        return np.mean(self.predictions, axis=1)\n    \n# Averaged & stacked models \naveraged_model = AveragingRegressor([catboost_model, xgb_model, rf_model, lgbm_model])\n\n\nstacked_model = StackingCVRegressor(\n    regressors=[catboost_model, xgb_model, rf_model, lgbm_model],\n    meta_regressor=Ridge()\n)\n\n# Test performance\ndef rmse_fun(predicted, actual):\n    return np.sqrt(np.mean(np.square(predicted - actual)))\n\nrmse = make_scorer(rmse_fun, greater_is_better=False)\n\nmodels = [\n     ('CatBoost', catboost_model),\n     ('XGBoost', xgb_model),\n     ('LightGBM', lgbm_model),\n     ('DecisionTree', tree_model),\n     ('RandomForest', rf_model),\n     ('Ridge', ridge_model),\n     ('Lasso', lasso_model),\n     ('KNN', knn_model),\n     ('Averaged', averaged_model),\n     ('Stacked', stacked_model),\n]\n\n\nscores = [\n    -1.0 * cross_val_score(model, train.values, train_y.values, scoring=rmse).mean()\n    for _,model in models\n]","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:29:53.485734Z","iopub.execute_input":"2022-02-25T01:29:53.485986Z","iopub.status.idle":"2022-02-25T01:39:04.431347Z","shell.execute_reply.started":"2022-02-25T01:29:53.485954Z","shell.execute_reply":"2022-02-25T01:39:04.430551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataz = pd.DataFrame({ 'Model': [name for name, _ in models], 'Error (RMSE)': scores })\ndataz.plot(x='Model', kind='bar')\nplt.savefig('stacked_scores.png')","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:39:04.432439Z","iopub.execute_input":"2022-02-25T01:39:04.432652Z","iopub.status.idle":"2022-02-25T01:39:04.857955Z","shell.execute_reply.started":"2022-02-25T01:39:04.432623Z","shell.execute_reply":"2022-02-25T01:39:04.856891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Results","metadata":{}},{"cell_type":"code","source":"dataz","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:39:04.859459Z","iopub.execute_input":"2022-02-25T01:39:04.859814Z","iopub.status.idle":"2022-02-25T01:39:04.870624Z","shell.execute_reply.started":"2022-02-25T01:39:04.859768Z","shell.execute_reply":"2022-02-25T01:39:04.869583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predictions","metadata":{}},{"cell_type":"code","source":"# Stacked model predictions (best score)\nstacked_model.fit(train.values, target.values)    \npredictions_stacked = stacked_model.predict(test.values)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:39:04.872388Z","iopub.execute_input":"2022-02-25T01:39:04.872619Z","iopub.status.idle":"2022-02-25T01:39:04.941564Z","shell.execute_reply.started":"2022-02-25T01:39:04.872589Z","shell.execute_reply":"2022-02-25T01:39:04.940625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Submission","metadata":{}},{"cell_type":"code","source":"# LightGBM/Xgboost\nsub_df = pd.read_csv('../input/sample_submission.csv')\nsub_df[\"target\"] = 0.5 * predictions_lgb + 0.5 * predictions_xgb\nsub_df.to_csv(\"submission_lgbxgboost.csv\", index=False)\n\n# Stacked\nsub_df = pd.read_csv('../input/sample_submission.csv')\nsub_df[\"target\"] = predictions_stacked\nsub_df.to_csv(\"submission_stacked.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T01:39:04.942628Z","iopub.status.idle":"2022-02-25T01:39:04.943100Z","shell.execute_reply.started":"2022-02-25T01:39:04.942915Z","shell.execute_reply":"2022-02-25T01:39:04.942934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}