{"cells":[{"metadata":{"_uuid":"2900dc913421919ce8786b33d054cfd898f97794"},"cell_type":"markdown","source":"## Outlier Detection and Removal"},{"metadata":{"_uuid":"d5f9e8594bec70c03b3be0c09ccd22b00631efe3"},"cell_type":"markdown","source":"### Abstract"},{"metadata":{"_uuid":"029d7184463319e052f74e4fb80326369cd832de"},"cell_type":"markdown","source":"This notebook contains two functions. \n\n1- print_quantile_info: Print out the following information about the data.\n\n2- remove_outliers_using_quantiles: Remove outliers according to the given fence value and return new dataframe."},{"metadata":{"trusted":true,"_uuid":"003db7d97a3e35ffc0a49c2bb565fb00f6ac1d8a"},"cell_type":"code","source":"import pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78d9ca50f10d91b1106e52c12f1f38b83c50eb21"},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"934ced584e060a6e1bf35209efad7cbed295b927"},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"797403abb7d3015d9c8645ee9e34dee2e0644543"},"cell_type":"code","source":"# Function: print_quantile_info(qu_dataset, qu_field)\n#   Print out the following information about the data\n#   - interquartile range\n#   - upper_inner_fence\n#   - lower_inner_fence\n#   - upper_outer_fence\n#   - lower_outer_fence\n#   - percentage of records out of inner fences\n#   - percentage of records out of outer fences\n# Input: \n#   - pandas dataframe (qu_dataset)\n#   - name of the column to analyze (qu_field)\n# Output:\n#   None\n\ndef print_quantile_info(qu_dataset, qu_field):\n    a = qu_dataset[qu_field].describe()\n    \n    iqr = a[\"75%\"] - a[\"25%\"]\n    print(\"interquartile range:\", iqr)\n    \n    upper_inner_fence = a[\"75%\"] + 1.5 * iqr\n    lower_inner_fence = a[\"25%\"] - 1.5 * iqr\n    print(\"upper_inner_fence:\", upper_inner_fence)\n    print(\"lower_inner_fence:\", lower_inner_fence)\n    \n    upper_outer_fence = a[\"75%\"] + 3 * iqr\n    lower_outer_fence = a[\"25%\"] - 3 * iqr\n    print(\"upper_outer_fence:\", upper_outer_fence)\n    print(\"lower_outer_fence:\", lower_outer_fence)\n    \n    count_over_upper = len(qu_dataset[qu_dataset[qu_field]>upper_inner_fence])\n    count_under_lower = len(qu_dataset[qu_dataset[qu_field]<lower_inner_fence])\n    percentage = 100 * (count_under_lower + count_over_upper) / a[\"count\"]\n    print(\"percentage of records out of inner fences: %.2f\"% (percentage))\n    \n    count_over_upper = len(qu_dataset[qu_dataset[qu_field]>upper_outer_fence])\n    count_under_lower = len(qu_dataset[qu_dataset[qu_field]<lower_outer_fence])\n    percentage = 100 * (count_under_lower + count_over_upper) / a[\"count\"]\n    print(\"percentage of records out of outer fences: %.2f\"% (percentage))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f635b1af02e9833b47eb25c0be0b42d0fbf2832"},"cell_type":"code","source":"print_quantile_info(df_train, \"target\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"64c61bba214a10a82c59c2eea294842f77fab00d"},"cell_type":"code","source":"# Function: remove_outliers_using_quantiles(qu_dataset, qu_field, qu_fence)\n#   1- Remove outliers according to the given fence value and return new dataframe.\n#   2- Print out the following information about the data\n#      - interquartile range\n#      - upper_inner_fence\n#      - lower_inner_fence\n#      - upper_outer_fence\n#      - lower_outer_fence\n#      - percentage of records out of inner fences\n#      - percentage of records out of outer fences\n# Input: \n#   - pandas dataframe (qu_dataset)\n#   - name of the column to analyze (qu_field)\n#   - inner (1.5*iqr) or outer (3.0*iqr) (qu_fence) values: \"inner\" or \"outer\"\n# Output:\n#   - new pandas dataframe (output_dataset)\n\ndef remove_outliers_using_quantiles(qu_dataset, qu_field, qu_fence):\n    a = qu_dataset[qu_field].describe()\n    \n    iqr = a[\"75%\"] - a[\"25%\"]\n    print(\"interquartile range:\", iqr)\n    \n    upper_inner_fence = a[\"75%\"] + 1.5 * iqr\n    lower_inner_fence = a[\"25%\"] - 1.5 * iqr\n    print(\"upper_inner_fence:\", upper_inner_fence)\n    print(\"lower_inner_fence:\", lower_inner_fence)\n    \n    upper_outer_fence = a[\"75%\"] + 3 * iqr\n    lower_outer_fence = a[\"25%\"] - 3 * iqr\n    print(\"upper_outer_fence:\", upper_outer_fence)\n    print(\"lower_outer_fence:\", lower_outer_fence)\n    \n    count_over_upper = len(qu_dataset[qu_dataset[qu_field]>upper_inner_fence])\n    count_under_lower = len(qu_dataset[qu_dataset[qu_field]<lower_inner_fence])\n    percentage = 100 * (count_under_lower + count_over_upper) / a[\"count\"]\n    print(\"percentage of records out of inner fences: %.2f\"% (percentage))\n    \n    count_over_upper = len(qu_dataset[qu_dataset[qu_field]>upper_outer_fence])\n    count_under_lower = len(qu_dataset[qu_dataset[qu_field]<lower_outer_fence])\n    percentage = 100 * (count_under_lower + count_over_upper) / a[\"count\"]\n    print(\"percentage of records out of outer fences: %.2f\"% (percentage))\n    \n    if qu_fence == \"inner\":\n        output_dataset = qu_dataset[qu_dataset[qu_field]<=upper_inner_fence]\n        output_dataset = output_dataset[output_dataset[qu_field]>=lower_inner_fence]\n    elif qu_fence == \"outer\":\n        output_dataset = qu_dataset[qu_dataset[qu_field]<=upper_outer_fence]\n        output_dataset = output_dataset[output_dataset[qu_field]>=lower_outer_fence]\n    else:\n        output_dataset = qu_dataset\n    \n    print(\"length of input dataframe:\", len(qu_dataset))\n    print(\"length of new dataframe after outlier removal:\", len(output_dataset))\n    \n    return output_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0194a36a4fa66bd1cc622d5d5173f19044281fe1"},"cell_type":"code","source":"df_train.dropna(inplace=True)\nnew_dataset = remove_outliers_using_quantiles(df_train, \"target\", \"inner\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7842aa279868e2903d1ee02594d0cc495890fbf6"},"cell_type":"code","source":"new_dataset.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f40d525f3832869d15fafbe98388a88c0e2dd48"},"cell_type":"markdown","source":"**Before outlier removal:**"},{"metadata":{"trusted":true,"_uuid":"243a77e9b6fb1648327b464210783fae0d749be8"},"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\nplt.figure(figsize=(15,10))\nsns.distplot(df_train[\"target\"]);\nplt.title('Histogram for Customer Loyalty Before Removal')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dcc38e4ff63f3743826ccca111679e7c10a1b995"},"cell_type":"markdown","source":"**After outlier removal:**"},{"metadata":{"trusted":true,"_uuid":"bfeeea77af66d579ec5c754655e4e427e4b3fcf4"},"cell_type":"code","source":"plt.figure(figsize=(15,10))\nsns.distplot(new_dataset[\"target\"]);\nplt.title('Histogram for Customer Loyalty After Removal')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}