{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Visualize feature dependency Pearson's Chi-square"},{"metadata":{},"cell_type":"markdown","source":"This kernel means to explore the dependency between discrete features. Proving reader a insight for workable direction."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.preprocessing import OrdinalEncoder,OneHotEncoder\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.model_selection import KFold,StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.preprocessing import StandardScaler\nimport string\nimport category_encoders as ce\nimport time","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Label encode all features and fill missing values with mode"},{"metadata":{"trusted":true},"cell_type":"code","source":"rawtrain=pd.read_csv('../input/cat-in-the-dat-ii/train.csv')\ntrain=rawtrain.drop('id',axis=1)\n\n#======encode ordinal\ncate_ord=['ord_1','ord_2']\nfor c in cate_ord:\n    print(rawtrain[c].unique())\nlevelmap={c:i for i,c in enumerate(['Novice','Contributor', 'Expert', 'Master','Grandmaster'])}\ntrain['ord_1']=train['ord_1'].replace(levelmap)\ntempratmap={c:i for i,c in enumerate(['Freezing','Cold', 'Warm','Hot' , 'Boiling Hot' ,'Lava Hot' ])}\ntrain['ord_2']=train['ord_2'].replace(tempratmap)\nlowermap={c:i for i,c in enumerate(string.ascii_lowercase)}\ntrain['ord_3']=train['ord_3'].replace(lowermap)\nupperletter=rawtrain['ord_4'].unique().tolist()\nupperletter.remove(np.nan)\nupperletter.sort()\nuppermap={c:i for i,c in enumerate(string.ascii_uppercase)}\ntrain['ord_4']=train['ord_4'].replace(uppermap)\n#/ord_5\nalletter=string.ascii_letters\nallmap={c:i for i,c in enumerate(alletter)}\ndef getP(x,p):\n    if pd.isnull(x):\n        return x\n    else:\n        if p==0:\n            return x[0]\n        else:\n            return x[1]\n        \ntrain['ord_5_0']=rawtrain['ord_5'].apply(lambda x: getP(x,0)).replace(allmap)\ntrain['ord_5_1']=rawtrain['ord_5'].apply(lambda x: getP(x,1)).replace(allmap)\ntrain=train.drop('ord_5',axis=1)\n#======encode binary and nominal+label to num for k mode clustering:https://www.kaggle.com/teejmahal20/clustering-categorical-data-k-modes-cat-ii\nnormcol59=['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']\ntrain_cluster=train.drop(normcol59,axis=1)\nfor c in train_cluster.columns:\n    train_cluster[c].fillna(train_cluster[c].mode()[0], inplace = True)\n\nbincol_labeled=['bin_3', 'bin_4']\nbinOE=OrdinalEncoder()\ntrain_cluster[bincol_labeled]=binOE.fit_transform(train_cluster[bincol_labeled])\n\nnormcol_labeled=['nom_0','nom_1','nom_2', 'nom_3', 'nom_4']\nbinOE=OrdinalEncoder()\ntrain_cluster[normcol_labeled]=binOE.fit_transform(train_cluster[normcol_labeled])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generate contengency table and result matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"#==========test independency\nimport scipy.stats as scs\n\ndef chi_square_of_df_cols(df, col1, col2):\n    df_col1, df_col2 = df[col1], df[col2]\n\n    result = [[sum((df_col1 == cat1) & (df_col2 == cat2))\n               for cat2 in df_col2.unique()]\n              for cat1 in df_col1.unique()]\n\n    return scs.chi2_contingency(result)\n\nchi_matrix=np.zeros([len(train_cluster.columns),len(train_cluster.columns)])\nfor i,r in enumerate(train_cluster.columns):\n    for j,c in enumerate(train_cluster.columns):\n        print('{}{}'.format(i,j),flush=True)\n        if i!=j:\n            stemp,tp,_,_=chi_square_of_df_cols(train_cluster, r, c)\n            chi_matrix[i,j]=tp\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualization using heatmap"},{"metadata":{"trusted":true},"cell_type":"code","source":"for i,r in enumerate(train_cluster.columns):\n    for j,c in enumerate(train_cluster.columns):\n        if i==j:\n            chi_matrix[i,j]=np.nan\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Notice the cell values in plot are p-values. Therefore the smaller the value the bigger the dependency."},{"metadata":{"trusted":true},"cell_type":"code","source":"f, ax = plt.subplots(figsize=(10, 10))\ncolormap = plt.cm.Greens_r\nsns.heatmap(pd.DataFrame(chi_matrix,columns=train_cluster.columns,index=train_cluster.columns), \n             cmap=colormap, square=True, linewidths=.5)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}