{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport math\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"history = pd.read_csv(\"../input/historical_transactions.csv\", low_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a534bb914717417f7043d7d46d4d37b8803a771a"},"cell_type":"code","source":"history.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"428b87398a3e9bfb6d36c549b4b97d9747177470"},"cell_type":"code","source":"history['purchase_amount'].hist(bins=100)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fb869494254f8cee69de5b21b964243a45b9ba5a"},"cell_type":"markdown","source":"As you can see we have outliers in this distribution we can confirm this conclusion by drawing the box plot "},{"metadata":{"_kg_hide-output":false,"trusted":true,"_uuid":"ddb04df9fd94d4af49db3d05326c6c39ff2033b0"},"cell_type":"code","source":"history['purchase_amount'].plot.box()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5d668f4530ffededcd46f266ed4c06ca3f5ea5f7"},"cell_type":"markdown","source":"we have point ( transaction) with purchase amount higer than 6000000. we try to eliminate it "},{"metadata":{"trusted":true,"_uuid":"dd8c62f64d42a5944857dd81de42d2a4c7c0bb8b"},"cell_type":"code","source":"history=history[history['purchase_amount']<5000000]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8657ba0900c68d840d2c331210d9641277849168"},"cell_type":"code","source":"history['purchase_amount'].hist()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cecd977e791bd7c4748799cbb3abae8d21cd6c68"},"cell_type":"markdown","source":"After elimination of that point, we still have outliers in this feature so, in the next few cells, I drew some histograms in order to allocate outliers, fill free to skip them and focus on the block final bolck  when I used **pd.cut **"},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"5513b521bb8fb640e9978e20ae21852db892fc3b"},"cell_type":"code","source":"history[history['purchase_amount']>25000]['purchase_amount'].hist()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"16615831fe8cede9abe63f2fd27c8f48b0e8bb22"},"cell_type":"code","source":"history[history['purchase_amount']<20000]['purchase_amount'].hist()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"f449bf4546cadfbc64cfc622d335eac8b4c24082"},"cell_type":"code","source":"history[history['purchase_amount']<2500]['purchase_amount'].hist()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"f8effd8bc8b7a14515c915491d5bf12d48536168"},"cell_type":"code","source":"history[history['purchase_amount']<500]['purchase_amount'].hist()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"512933511511f0caa378798359d796d41cf1f77d"},"cell_type":"code","source":"history[history['purchase_amount']<100]['purchase_amount'].hist()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"0242017d1c2178e8dbb8ab05d330558a74832ea7"},"cell_type":"code","source":"history[history['purchase_amount']<10]['purchase_amount'].hist()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"dde5a43c44b678b7703fba1fbb58be7727cd2819"},"cell_type":"code","source":"history[history['purchase_amount']<1]['purchase_amount'].hist()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"ce9b4dc2a61cadda63194f90e4bd229d00cc43de"},"cell_type":"code","source":"history[history['purchase_amount']<1].shape","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"4dfe96153934bb0c3fcb6e59a1d3c82ddc5c6dd2"},"cell_type":"code","source":"history[history['purchase_amount']<1].shape[0]/history.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"dc9059714ab8886ff24d90c9897472413998692f"},"cell_type":"code","source":"history[history['purchase_amount']>1]['purchase_amount'].hist()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"4a0d874fb327373d2e8fa3038426bf1210e8a484"},"cell_type":"code","source":"history[(history['purchase_amount']>1) & (history['purchase_amount']<10)]['purchase_amount'].hist()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"aa8bcdfd1d18763ce1c6bdd2d0d2b64b1107b4ac"},"cell_type":"code","source":"history[(history['purchase_amount']>10) & (history['purchase_amount']<100)]['purchase_amount'].hist()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"e9cb8dacd9adaeee611a36bd83789086f2795e50"},"cell_type":"code","source":"history[(history['purchase_amount']>100) & (history['purchase_amount']<1000)]['purchase_amount'].hist()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"f60a26ccae58767931fa2b08412d7b96b3e7b263"},"cell_type":"code","source":"history[(history['purchase_amount']>10000) & (history['purchase_amount']<100000)]['purchase_amount'].hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ab357bfaf156c53eeb074bdeacb463f0b855b8d"},"cell_type":"code","source":"bins = [-1,1,10,100,1000,10000,100000,1000000,10000000]\nhistory['binned_purchase_amount'] = pd.cut(history['purchase_amount'], bins)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8147a59f0c1f5dbd13fc3d74394140ba5d723786"},"cell_type":"code","source":"binned_purchase_amount_cnt = history.groupby(\"binned_purchase_amount\")['binned_purchase_amount'].count().reset_index(name='binned_purchase_amount_cnt')\nbinned_purchase_amount_cnt.columns = ['purchase_amount','binned_purchase_amount_cnt']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"21b9f082224925046fd26c408a632703e47547ee"},"cell_type":"code","source":"binned_purchase_amount_cnt['percent']=binned_purchase_amount_cnt['binned_purchase_amount_cnt']*100/binned_purchase_amount_cnt['binned_purchase_amount_cnt'].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0173d68e368a287c34d1d810c8737723e4292d17"},"cell_type":"code","source":"binned_purchase_amount_cnt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"30a7d87103b4ccaafb9725592b2148b3b4d6d376"},"cell_type":"markdown","source":"As you can see more than 98% of this distribution still between -1 and 1, we define data outside this interval as  outliers."},{"metadata":{"trusted":true,"_uuid":"4f2057c4c704bafcde8040f81e718bbe4041defe"},"cell_type":"code","source":"binned_purchase_amount_cnt.plot.bar(y='percent', x='purchase_amount')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"216ae6cf86f81880cd8986faef4ae72ad9157da3"},"cell_type":"markdown","source":"In order to reduce variance and keep the sign of the distribution, I used this transformation : sign(x)*ln(1+abs(x))"},{"metadata":{"trusted":true,"_uuid":"2c60074a0a92561695b11b6dc7d01493424c9d67"},"cell_type":"code","source":"def log_1(x):\n    return math.copysign(1,x)*math.log(1+abs(x))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6f4fa1b305510793feedda7705b46bfab237cb6"},"cell_type":"code","source":"history['log_purchase_amount']=history['purchase_amount'].apply(lambda x :log_1(x) )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e65c6d23210f713a7ac6abefa2046c0ea597aa41"},"cell_type":"code","source":"history['log_purchase_amount'].hist()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fb3c74ac731cfcea55c713015512b948e09f4985"},"cell_type":"markdown","source":"instead of eliminating outliers, it may be useful to score the loyalty of the customer, I create a new feature to distinguish between normal data and outliers."},{"metadata":{"trusted":true,"_uuid":"86fa9ff103c2b7938767fbde8311ce66815e9035"},"cell_type":"code","source":"history['purchase_amount_outliers'] = 0\nhistory.loc[history['purchase_amount'] >1, 'purchase_amount_outliers'] = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"064b5bf48081cea70f0ab58e5424a222ded053f8"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}