{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom tqdm import tqdm\ntqdm.pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a948cd49aa0f82c37fe6d0583292f27dea15300b"},"cell_type":"code","source":"data = pd.read_csv('../input/elo-merchant-eda/data_train.csv.gz')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f74144a6ec1b823f43ff2bf950f0ebf2958e8902"},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cfbe0b148d584007a4c4103220f9463d53376f0"},"cell_type":"code","source":"data = data.sort_values(by='first_active_month')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b95f35f1c3742dff291f07be1697de8299fbc79b"},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9e2d33c3a664c94e8bdf114f2f37f8cab4150a3"},"cell_type":"code","source":"print('The earliest date is {}, most recent date is {}.'.format(data.first_active_month.iloc[0], data.first_active_month.iloc[-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78739253a99bdee5a8a2c77b55586c63c487dc2d"},"cell_type":"code","source":"data.target.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cbbab60186a4391e3f4b72f3e8fb2fe21a631990"},"cell_type":"code","source":"target_stats = data.groupby('first_active_month').target.agg(['count','max','min','mean','median'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8e10928c03e058b9970b862d5d1e05579c9e452"},"cell_type":"code","source":"target_stats.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2692504571168ca9ca27fad738b8c6f0c729f6a1"},"cell_type":"code","source":"fig, ax = plt.subplots()\nax.plot(target_stats.iloc[:,0])\nax.set_xlabel(xlabel='First Active Month')\nax.set_ylabel(ylabel='Frequency')\nax.xaxis.set_tick_params(rotation=90)\nstart, end = ax.get_xlim()\nax.xaxis.set_ticks(np.arange(start, end, 5))\nax.set_title('Frequency of target in chronological order')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9f9829d5a9f99ea6f524a33e52938775b98ba6b"},"cell_type":"code","source":"fig, ax = plt.subplots()\nax.plot(target_stats.iloc[:,[1,2]])\nax.set_xlabel(xlabel='First Active Month')\nax.set_ylabel(ylabel='Target')\nax.xaxis.set_tick_params(rotation=90)\nstart, end = ax.get_xlim()\nax.xaxis.set_ticks(np.arange(start, end, 5))\nax.set_title('Max and min of target in chronological order')\nax.legend(['Max','Min'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9b765bd1cee20b5de6f3438d09fe420df5ba74d6"},"cell_type":"markdown","source":"From the above plot, we can see that in almost all the month, there will be a loyalty score of less than -30."},{"metadata":{"trusted":true,"_uuid":"66f79a5ec70a60e025086427f7a9f03e69b2f8a2"},"cell_type":"code","source":"fig, ax = plt.subplots()\nax.plot(target_stats.iloc[:,[3]])\nax.set_xlabel(xlabel='First Active Month')\nax.set_ylabel(ylabel='Target')\nax.xaxis.set_tick_params(rotation=90)\nstart, end = ax.get_xlim()\nax.xaxis.set_ticks(np.arange(start, end, 5))\nax.set_title('Mean of target in chronological order')\nax.legend(['Mean'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c30084cbd48418fefab7de980ddf639b45c00a98"},"cell_type":"code","source":"fig, ax = plt.subplots()\nax.plot(target_stats.iloc[:,[4]])\nax.set_xlabel(xlabel='First Active Month')\nax.set_ylabel(ylabel='Target')\nax.xaxis.set_tick_params(rotation=90)\nstart, end = ax.get_xlim()\nax.xaxis.set_ticks(np.arange(start, end, 5))\nax.set_title('Median of target in chronological order')\nax.legend(['Median'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"48e0ac3a2adce50a4ef8611816c3b8545278cb79"},"cell_type":"markdown","source":"Let's investigate these less than -30 loyalty scores."},{"metadata":{"trusted":true,"_uuid":"eef35bb1a0e7cd15e836d16b5120d70c1992a241"},"cell_type":"code","source":"less_than_thirty = data.loc[data.target < -30]\nless_than_thirty.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0159f35867a164dd6905491f053b9bb13b58882d"},"cell_type":"code","source":"print('There are {} records with loyalty score less than -30.'.format(less_than_thirty.shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a15ee9237e440f1ac66382d093b94961c89c3e0"},"cell_type":"code","source":"less_than_thirty.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dbfc151f5885dd8ec36c469f772f99b49887b0ce"},"cell_type":"code","source":"less_than_thirty.target.unique()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"111767316a168f4e2ffcd33762d95187fdd11774"},"cell_type":"markdown","source":"There is only one value for this loyalty score."},{"metadata":{"trusted":true,"_uuid":"bd69db10a510b83e7950e28b56f6f5387e44861f"},"cell_type":"code","source":"target_stats_less_than_thirty = less_than_thirty.groupby('first_active_month').target.agg(['count'])\ntarget_stats_less_than_thirty.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd58fbd845cb1018ebc301a13bd0d1bf3aaca3c6"},"cell_type":"code","source":"fig, ax = plt.subplots()\nax.plot(target_stats_less_than_thirty.iloc[:,0])\nax.set_xlabel(xlabel='First Active Month')\nax.set_ylabel(ylabel='Frequency')\nax.xaxis.set_tick_params(rotation=90)\nstart, end = ax.get_xlim()\nax.xaxis.set_ticks(np.arange(start, end, 5))\nax.set_title('Frequency of target in chronological order')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d4866969dfa22c1693993eb16106e89740465f89"},"cell_type":"markdown","source":"The distribution seems to follow the general distribution of the whole training sample."},{"metadata":{"trusted":true,"_uuid":"e1c35fdbe0ec331beb9f9f4b3646885934847f6e"},"cell_type":"code","source":"target_stats = target_stats.join(target_stats_less_than_thirty, rsuffix='_less_than_thirty')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc61e70ffb9d022f8b1ef6c81fa5246f94b05e61"},"cell_type":"code","source":"target_stats = target_stats.fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab520326db0cd27a35b51fa1a6c0ecba307c13df"},"cell_type":"code","source":"target_stats.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab413366fc44db0aec66b018d91c64d1396f3ef4"},"cell_type":"code","source":"target_stats['percent_of_less_than_thirty'] = target_stats.iloc[:,-1]/target_stats.iloc[:,0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"363af6c6590f50ec50335143a07cbac4fa20d1ca"},"cell_type":"code","source":"target_stats.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dabffa1843bf3c32326a26cbac078d78b8eca45a"},"cell_type":"code","source":"target_stats.percent_of_less_than_thirty.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e367233c9881408e7aed66cabc7502e4daeeceb2"},"cell_type":"code","source":"fig, ax = plt.subplots()\nax.plot(target_stats.iloc[:,-1])\nax.set_xlabel(xlabel='First Active Month')\nax.set_ylabel(ylabel='Percentage')\nax.xaxis.set_tick_params(rotation=90)\nstart, end = ax.get_xlim()\nax.xaxis.set_ticks(np.arange(start, end, 5))\nax.set_title('% target which is < -30 in chronological order')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"962ca419742ee8e39b48f75ce980de2963eaa1a2"},"cell_type":"markdown","source":"It looks like the percentage of target that is less than -30 is quite stable, with a slight decrease across the months."},{"metadata":{"trusted":true,"_uuid":"c32094e37fb5d5b551abd3b55287bb47a5a36b05"},"cell_type":"code","source":"data['feature_combn'] = data.progress_apply(lambda x: str([x.feature_1,x.feature_2, x.feature_3]), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78a429a7c37db0df6b5a6a021fba918a946a5865"},"cell_type":"code","source":"print('There are {} feature combinations.'.format(len(data.feature_combn.unique())))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b48d1cea78dc6c5f53f46b5f39743a13481a2eb2"},"cell_type":"code","source":"len(data.feature_1.unique())*len(data.feature_2.unique())*len(data.feature_3.unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e72d3ba6891f4e669cb243799eb9f993285273e"},"cell_type":"code","source":"pd.crosstab(index=data.feature_1, columns=data.feature_2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3fc11b73832ceb7a22b17af3dd7bde1c0d38d69c"},"cell_type":"code","source":"pd.crosstab(index=data.feature_1, columns=data.feature_3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e246af71f6762e8673c8d2a4bac0033f1dacd49"},"cell_type":"code","source":"pd.crosstab(index=data.feature_2, columns=data.feature_3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d6c33374361b966b053ed670254633b59c638fb"},"cell_type":"code","source":"feature_combn_target_stats = data.groupby('feature_combn').target.agg(['count','max','min','mean','median'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e63eb88aae966aa4d8fd8e82a0fde5fa5fbd879"},"cell_type":"code","source":"feature_combn_target_stats","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9aa0a3476eb6936e94f07a45ac3ac5450325d2bb"},"cell_type":"markdown","source":"From the table, we can see that regardless of feature combination, the min loyalty score is still -33.219."},{"metadata":{"trusted":true,"_uuid":"5bde220a1df568036c2e9240e344d17d90dd51af"},"cell_type":"code","source":"data_test = pd.read_csv('../input/elo-merchant-eda/data_test.csv.gz')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c0df05865755fe46bf1f5728e8ee54fb86b6670"},"cell_type":"code","source":"data_test['feature_combn'] = data_test.progress_apply(lambda x: str([x.feature_1,x.feature_2, x.feature_3]), axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85601a384e7f2ae2148559b82037b33e771b92ee"},"cell_type":"code","source":"print('There are {} feature combinations.'.format(len(data_test.feature_combn.unique())))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31956ef6252a2948cb2435bfdaf243a5c8146d5a"},"cell_type":"code","source":"pd.crosstab(index=data_test.feature_1, columns=data_test.feature_2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2258c9aa6cc32fc2b1abec78a542c3ca31618216"},"cell_type":"code","source":"pd.crosstab(index=data_test.feature_1, columns=data_test.feature_3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83420bcf504284d3670c8b8ffca4e64d6f7ba81a"},"cell_type":"code","source":"pd.crosstab(index=data_test.feature_2, columns=data_test.feature_3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f98c6fdc6a245f840464ddc903da9a12f24273a5"},"cell_type":"markdown","source":"Test set display the same relationships for features."},{"metadata":{"trusted":true,"_uuid":"27ffdc482416cde3af4cac420eaf71ea5aeb47a6"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}