{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\nimport os\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom pandas_profiling import ProfileReport\n\nimport matplotlib as mplot\nfrom matplotlib import pyplot as plt\nfrom mpl_toolkits import mplot3d\nfrom matplotlib import colors\n\nbase_colors_values = list(colors.BASE_COLORS.values())\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/fabiendaniel/elo-world\ndef reduce_mem_usage(df, verbose=True):\n    prefixes = ['int', 'float']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = str(df[col].dtype)\n        if not col_type.startswith('int') and not col_type.startswith('float'):\n            print('col_type:', col_type, 'not compressed')\n            continue\n        c_min = df[col].min()\n        c_max = df[col].max()\n        if col_type.startswith('int'):\n            if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                df[col] = df[col].astype(np.int8)\n            elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                df[col] = df[col].astype(np.int16)\n            elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                df[col] = df[col].astype(np.int32)\n            elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                df[col] = df[col].astype(np.int64)  \n        elif col_type.startswith('float'):\n            if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                df[col] = df[col].astype(np.float16)\n            elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                df[col] = df[col].astype(np.float32)\n            else:\n                df[col] = df[col].astype(np.float64)    \n    if verbose:\n        end_mem = df.memory_usage().sum() / 1024**2\n        print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"from pathlib import Path\nINPUT = Path('../input/elo-merchant-category-recommendation/')\ntrain = pd.read_csv(INPUT/'train.csv')\ntest = pd.read_csv(INPUT/'test.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Columns:', *train.columns, sep=' | ')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = train.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/sudalairajkumar/simple-exploration-notebook-elo\ntarget_col = \"target\"\n\nplt.figure(figsize=(8,6))\nplt.scatter(range(df_train.shape[0]), np.sort(df_train[target_col].values))\nplt.xlabel('index', fontsize=12)\nplt.ylabel('Loyalty Score', fontsize=12)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axs = plt.subplots()\n\naxs.hist(df_train.target, range(-34, 19), color='r')\naxs.set_title('Hist of Loyalty', color='g');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Extremely low scores (< -30):\", (df_train.target < -30).sum())\nprint(\"Extremely HIGH scores(> 10):\", (df_train.target > 10).sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots()\nax.hist(df_train.target, range(-10, 10), color='purple')\nax.set_title('[-10, 10] Hist of Loyalty', color='g');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots()\nax.hist(df_train.target, range(-3, 4), color='cyan', density=True)\nax.set_title('[-3, 3] Hist of Loyalty', color='blue',);","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"first_mount = df_train.first_active_month.value_counts()\nsrt = first_mount.sort_index()\nyears = srt.index.str[:4].unique()\n\n# WHITE color doesn't well... appear\nMY_BASE_COLORS = colors.BASE_COLORS.copy()\ndel MY_BASE_COLORS['w']\nMY_BASE_COLORS['purple'] = 'purple'\n\nyear_cmap = dict(zip(years, MY_BASE_COLORS))\n\ncmap_seq = srt.index.map(lambda x: year_cmap[x[:4]])\n\n\nfig = plt.figure(figsize=(14, 10))\n\nplt.bar(\n    srt.index,\n    srt.values,\n    color=cmap_seq,\n)\nplt.xticks(rotation='vertical');\nplt.xlabel('First month');\nplt.title('First card active month', color='g');\n\nax = plt.gca()\nax.set_facecolor((.6, .44, .98, .6))\n# fig.patch.set_facecolor('xkcd:mint green')\nfor i, t in enumerate(plt.gca().get_xticklabels()):\n    t.set_color( cmap_seq[i] )\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"first_month = df_train.first_active_month.value_counts()\nsrt = first_month.sort_index()\nyears = srt.index.str[:4].unique()\nyear_cmap = dict(zip(years, MY_BASE_COLORS))\n\nvc = df_train.first_active_month.str[:4].value_counts()\nsrt=vc.sort_index()\n\nindices = np.arange(len(srt))\nfig, ax = plt.subplots(figsize=(12.8, 12.6))\nax.pie(\n    srt.values,\n    labels=srt.index\n)\nax.set_title('First-month YEAR active', fontdict={'color': 'red'});\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_=pd.cut(\n    df_train.target,\n    range(-34, 19)\n)\nvc = _.value_counts()\nsrt=vc.sort_index()\n\nindices = np.arange(len(srt))\nfig, ax = plt.subplots(figsize=(12.8, 12.6))\nax.pie(\n    srt.values,\n    labels=srt.index\n)\nax.set_title('Loyalty Score Bins distribution', fontdict={'color': 'red'});","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fts = [f'feature_{i}' for i in range(1,4)]\nfig, ax = plt.subplots()\n\nfor ft in fts:\n    _ = train.groupby(ft).size().sort_index()\n    ax.bar(_.index, _.values)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fts = [f'feature_{i}' for i in range(1,4)]\nfig, axs = plt.subplots(nrows=1, ncols=len(fts), figsize=(len(fts) * 5 + 1, 8),\n                        edgecolor='black',\n                       frameon=True\n)\n\n# fig = plt.figure()\n# ax = fig.add_subplot()\nfor i, ft in enumerate(fts):\n#     _ = train.groupby(ft).size().sort_index()\n    _ = train.groupby(ft).size().sort_values()\n    \n#     ax.bar(_.index, _.values, tick_label=_.index, color=base_colors_values[i])    \n    axs[i].pie(_.values, labels=_.index, autopct='%1.1f%%')\n    axs[i].set_title(ft, color=base_colors_values[i])\n#     axs[i].bar(_.index, _.values, tick_label=_.index, color=base_colors_values[i])\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# [time consuming] Don't run cell bellow unless it hasn't be once.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_historical = pd.read_csv(INPUT/'historical_transactions.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_historical.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_historical.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_cols = [\n    'city_id', 'category_1', 'installments',\n    'merchant_category_id', 'merchant_id', 'month_lag',\n    'category_2', 'state_id', 'subsector_id'\n]\n\nfor c in cat_cols:\n    print(c, df_historical[c].unique())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base_colors_values = list(colors.BASE_COLORS.values())\n\nn = 4\nfig, axs = plt.subplots(4, 1, figsize=(15, n * 5 + 1))\nfor i, cat in enumerate(['category_2', 'installments', 'state_id', 'category_1']):\n    \n    _ = df_historical.groupby(by=cat).size().sort_values()\n    if cat == 'installments':\n        _.drop(999, inplace=True)\n    c=base_colors_values[i]\n    axs[i].bar(_.index, _.values, color=c)\n    axs[i].set_title(cat, color=c, loc='right', size='large', style='italic', weight='heavy')\n    maxval = max(_.values)\n    for idx, xval in enumerate(_.values):\n        rotation = 45 * (len(_.index) > 20)\n        rotation += 45        \n        axs[i].text(\n            _.index[idx], 1.07 * xval, str(xval),\n            rotation = rotation if xval < maxval else 0,\n            size='medium',\n            weight='bold',\n            color='black',\n        )\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_historical.groupby('installments').size().drop(999)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"r=df_historical.groupby('installments').size().drop(999)\n# plt.bar(r.index[:3], r.values[:3]);\nplt.bar(r.index, r.values, color='red');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_historical.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}