{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport scipy.stats as ss\nimport matplotlib.pyplot as plt\nfrom sklearn import preprocessing\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\n\n#path_data = \"/kaggle/input/cat-in-the-dat-ii/\"\n#path_train = path_data + \"train.csv\"\n#df = pd.read_csv(path_train)\n\ntrain_I = pd.read_csv(\"/kaggle/input/cat-in-the-dat/train.csv\", index_col = \"id\")\ntrain_II = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/train.csv\", index_col = \"id\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#From https://towardsdatascience.com/the-search-for-categorical-correlation-a1cf7f1888c9\ndef cramers_v(confusion_matrix):\n    \"\"\" calculate Cramers V statistic for categorial-categorial association.\n        uses correction from Bergsma and Wicher,\n        Journal of the Korean Statistical Society 42 (2013): 323-328\n    \"\"\"\n    chi2 = ss.chi2_contingency(confusion_matrix)[0]\n    n = confusion_matrix.sum()\n    phi2 = chi2 / n\n    r, k = confusion_matrix.shape\n    phi2corr = max(0, phi2 - ((k-1)*(r-1))/(n-1))\n    rcorr = r - ((r-1)**2)/(n-1)\n    kcorr = k - ((k-1)**2)/(n-1)\n    return np.sqrt(phi2corr / min((kcorr-1), (rcorr-1)))\n\ndef cramers_v_matrix(df,cols):\n    result=pd.DataFrame()\n    for col1 in cols:\n        corr_list=[]\n        for col2 in cols:\n            confusion_matrix = pd.crosstab(df[col1], df[col2]).values\n            cramers=cramers_v(confusion_matrix)\n            if col1==col2:\n                cramers=0\n            corr_list.append(cramers)\n        df_temp = pd.DataFrame(corr_list, index =cols,columns =[col1]) \n        result = pd.concat([result, df_temp], axis=1, sort=False)\n    return result\n\ndef summary(df):\n    summary = pd.DataFrame(df.dtypes, columns=['dtypes'])\n    summary = summary.reset_index()\n    summary['Name'] = summary['index']\n    summary = summary[['Name', 'dtypes']]\n    summary['Missing'] = df.isnull().sum().values    \n    summary['Uniques'] = df.nunique().values\n    return summary\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_I.dropna(inplace=True)\ntrain_II.dropna(inplace=True)\nsummary_data_Cat_I=summary(train_I)\nsummary_data_Cat_II=summary(train_II)\ncols_Cat_I=summary_data_Cat_I[(summary_data_Cat_I.Uniques <27) & (summary_data_Cat_I.Name!=\"target\")]['Name'].to_list()\ncols_Cat_II=summary_data_Cat_II[(summary_data_Cat_II.Uniques <27) & (summary_data_Cat_II.Name!=\"target\")]['Name'].to_list()\ncramers_matrix_I=cramers_v_matrix(train_I,cols_Cat_I)\ncramers_matrix_II=cramers_v_matrix(train_II,cols_Cat_II)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Poor correlation coefficient. Values updated just for Plotting.\nresult=cramers_matrix_I.round(4)\ncols=cols_Cat_I\n\nfig = plt.figure(figsize=(30, 15))\n\nax = fig.add_subplot(111)\nax.set_title('Cat_I')\nplt.imshow(result)\nax.set_aspect('equal')\n\n# We want to show all ticks...\nax.set_xticks(np.arange(len(cols)))\nax.set_yticks(np.arange(len(cols)))\n# ... and label them with the respective list entries\nax.set_xticklabels(cols)\nax.set_yticklabels(cols)\n\n# Rotate the tick labels and set their alignment.\nplt.setp(ax.get_xticklabels(), rotation=45, ha=\"right\",\n         rotation_mode=\"anchor\")\n\n# Loop over data dimensions and create text annotations.\nfor i in range(len(cols)):\n    for j in range(len(cols)):\n        text = ax.text(j, i, result.iloc[i, j],\n                       ha=\"center\", va=\"center\", color=\"w\")\n\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Poor correlation coefficient. Values updated just for Plotting.\nresult=cramers_matrix_II.round(4)\ncols=cols_Cat_II\n\nfig = plt.figure(figsize=(30, 15))\n\nax = fig.add_subplot(111)\nax.set_title('Cat_II')\nplt.imshow(result)\nax.set_aspect('equal')\n\n# We want to show all ticks...\nax.set_xticks(np.arange(len(cols)))\nax.set_yticks(np.arange(len(cols)))\n# ... and label them with the respective list entries\nax.set_xticklabels(cols)\nax.set_yticklabels(cols)\n\n# Rotate the tick labels and set their alignment.\nplt.setp(ax.get_xticklabels(), rotation=45, ha=\"right\",\n         rotation_mode=\"anchor\")\n\n# Loop over data dimensions and create text annotations.\nfor i in range(len(cols)):\n    for j in range(len(cols)):\n        text = ax.text(j, i, result.iloc[i, j],\n                       ha=\"center\", va=\"center\", color=\"w\")\n\n\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}