{"cells":[{"metadata":{},"cell_type":"markdown","source":"**This notebook is going to EDA and create data visualization.**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Overview the dataset**"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/cat-in-the-dat-ii/train.csv\")\nprint(train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.dtypes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The data contains binary features (bin_), nominal features (nom_), ordinal features (ord_) as well as (potentially cyclical) day (of the week) and month features. The string ordinal features ord_{3-5} are lexically ordered according to string.ascii_letters."},{"metadata":{"trusted":true},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Distribution of each variable**"},{"metadata":{"trusted":true},"cell_type":"code","source":"#The distribution of \"target\"\nplt.figure(figsize = (5, 5))\nax = sns.countplot(x=\"target\", data=train, palette=\"Set2\")\nax.set_title('Target Distribution', weight = 'bold',fontsize = 15)\nplt.xlabel('Target', fontsize = 12)\nplt.ylabel('Count', fontsize = 12)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Binary Features (bin_)"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(3,2,figsize = (15,20))\nfor i in range(5): \n    sns.countplot(f'bin_{i}', data= train,ax = ax[i//2][i%2])\n    ax[i//2][i%2].set_ylim([0, 550000])\nfig.suptitle(\"Binary Feature Distribution\", fontsize=20, weight = 'bold')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Nominal features (nom_)"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(3,2, figsize=(15, 20))\nfor i in range(5): \n    sns.countplot(f'nom_{i}', data= train, ax=ax[i//2][i%2],\n                 order=train[f'nom_{i}'].value_counts().index)\n    ax[i//2][i%2].set_ylim([0, 350000])\nfig.suptitle(\"Nominal Feature Distribution\", fontsize = 20, weight = 'bold')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Ordinal features (ord_) "},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(3,2, figsize=(15, 20))\nfor i in range(5): \n    sns.countplot(f'ord_{i}', data= train, ax=ax[i//2][i%2],\n                 order=train[f'ord_{i}'].value_counts().index)\n    ax[i//2][i%2].set_ylim([0, 250000])\nfig.suptitle(\"Ordinal Feature Distribution\", fontsize = 20, weight = 'bold')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Day and Month"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(2,1, figsize=(24, 16))\n\nsns.countplot('day', hue='target', data= train, ax=ax[0], palette=\"Set2\")\nax[0].set_ylim([0, 100000])\n\nsns.countplot('month', hue='target', data= train, ax=ax[1], palette=\"Set3\")\nax[1].set_ylim([0, 70000])\n\nfig.suptitle(\"Time Distribution\", fontsize=20, weight = 'bold')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Find correlation"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.groupby('bin_3').count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.groupby('bin_4').count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['bin_3']=np.where(train['bin_3'] =='F', '0', train['bin_3'])\ntrain['bin_3']=np.where(train['bin_3'] =='T', '1', train['bin_3'])\n\ntrain['bin_4']=np.where(train['bin_4'] =='N', '0', train['bin_4'])\ntrain['bin_4']=np.where(train['bin_4'] =='Y', '1', train['bin_4'])\n\nbin = train[['id','bin_0','bin_1','bin_2','bin_3','bin_4','target']]\nbin[\"bin_3\"]= bin[\"bin_3\"].astype(float) \nbin[\"bin_4\"]= bin[\"bin_4\"].astype(float) \nbin.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bin = bin.interpolate()\nprint(bin.isnull().sum())\n\ncorrelation_b = bin.corr()\nplt.figure(figsize = (10, 10))\nsns.set(font_scale = 1.5)\nsns.heatmap(correlation_b, annot = True, annot_kws = {'size': 10}, cmap = 'Blues')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.groupby('nom_0').count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['nom_0']=np.where(train['nom_0'] =='Blue', '1', train['nom_0'])\ntrain['nom_0']=np.where(train['nom_0'] =='Green', '2', train['nom_0'])\ntrain['nom_0']=np.where(train['nom_0'] =='Red', '3', train['nom_0'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.groupby('nom_1').count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['nom_1']=np.where(train['nom_1'] =='Circle', '1', train['nom_1'])\ntrain['nom_1']=np.where(train['nom_1'] =='Polygon', '2', train['nom_1'])\ntrain['nom_1']=np.where(train['nom_1'] =='Square', '3', train['nom_1'])\ntrain['nom_1']=np.where(train['nom_1'] =='Star', '4', train['nom_1'])\ntrain['nom_1']=np.where(train['nom_1'] =='Trapezoid', '5', train['nom_1'])\ntrain['nom_1']=np.where(train['nom_1'] =='Triangle', '6', train['nom_1'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.groupby('nom_2').count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['nom_2']=np.where(train['nom_2'] =='Axolotl', '1', train['nom_2'])\ntrain['nom_2']=np.where(train['nom_2'] =='Cat', '2', train['nom_2'])\ntrain['nom_2']=np.where(train['nom_2'] =='Dog', '3', train['nom_2'])\ntrain['nom_2']=np.where(train['nom_2'] =='Hamster', '4', train['nom_2'])\ntrain['nom_2']=np.where(train['nom_2'] =='Lion', '5', train['nom_2'])\ntrain['nom_2']=np.where(train['nom_2'] =='Snake', '6', train['nom_2'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.groupby('nom_3').count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['nom_3']=np.where(train['nom_3'] =='Canada', '1', train['nom_3'])\ntrain['nom_3']=np.where(train['nom_3'] =='China', '2', train['nom_3'])\ntrain['nom_3']=np.where(train['nom_3'] =='Costa Rica', '3', train['nom_3'])\ntrain['nom_3']=np.where(train['nom_3'] =='Finland', '4', train['nom_3'])\ntrain['nom_3']=np.where(train['nom_3'] =='India', '5', train['nom_3'])\ntrain['nom_3']=np.where(train['nom_3'] =='Russia', '6', train['nom_3'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.groupby('nom_4').count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['nom_4']=np.where(train['nom_4'] =='Bassoon', '1', train['nom_4'])\ntrain['nom_4']=np.where(train['nom_4'] =='Oboe', '2', train['nom_4'])\ntrain['nom_4']=np.where(train['nom_4'] =='Piano', '3', train['nom_4'])\ntrain['nom_4']=np.where(train['nom_4'] =='Theremin', '4', train['nom_4'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nom = train[['id','nom_0','nom_1','nom_2','nom_3','nom_4','target']]\nnom[\"nom_0\"]= nom[\"nom_0\"].astype(float) \nnom[\"nom_1\"]= nom[\"nom_1\"].astype(float) \nnom[\"nom_2\"]= nom[\"nom_2\"].astype(float) \nnom[\"nom_3\"]= nom[\"nom_3\"].astype(float) \nnom[\"nom_4\"]= nom[\"nom_4\"].astype(float)\nnom.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nom = nom.interpolate()\nprint(nom.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"correlation_n = nom.corr()\nplt.figure(figsize = (10, 10))\nsns.set(font_scale = 1.5)\nsns.heatmap(correlation_n, annot = True, annot_kws = {'size': 10}, cmap = 'Reds')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.groupby('ord_2').count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['ord_2']=np.where(train['ord_2'] =='Boiling Hot', '1', train['ord_2'])\ntrain['ord_2']=np.where(train['ord_2'] =='Cold', '2', train['ord_2'])\ntrain['ord_2']=np.where(train['ord_2'] =='Freezing', '3', train['ord_2'])\ntrain['ord_2']=np.where(train['ord_2'] =='Hot', '4', train['ord_2'])\ntrain['ord_2']=np.where(train['ord_2'] =='Lava Hot', '5', train['ord_2'])\ntrain['ord_2']=np.where(train['ord_2'] =='Warm', '6', train['ord_2'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.groupby('ord_3').count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['ord_3']=np.where(train['ord_3'] =='a', '1', train['ord_3'])\ntrain['ord_3']=np.where(train['ord_3'] =='b', '2', train['ord_3'])\ntrain['ord_3']=np.where(train['ord_3'] =='c', '3', train['ord_3'])\ntrain['ord_3']=np.where(train['ord_3'] =='d', '4', train['ord_3'])\ntrain['ord_3']=np.where(train['ord_3'] =='e', '5', train['ord_3'])\ntrain['ord_3']=np.where(train['ord_3'] =='f', '6', train['ord_3'])\ntrain['ord_3']=np.where(train['ord_3'] =='g', '7', train['ord_3'])\ntrain['ord_3']=np.where(train['ord_3'] =='h', '8', train['ord_3'])\ntrain['ord_3']=np.where(train['ord_3'] =='i', '9', train['ord_3'])\ntrain['ord_3']=np.where(train['ord_3'] =='j', '10', train['ord_3'])\ntrain['ord_3']=np.where(train['ord_3'] =='k', '11', train['ord_3'])\ntrain['ord_3']=np.where(train['ord_3'] =='l', '12', train['ord_3'])\ntrain['ord_3']=np.where(train['ord_3'] =='m', '13', train['ord_3'])\ntrain['ord_3']=np.where(train['ord_3'] =='n', '14', train['ord_3'])\ntrain['ord_3']=np.where(train['ord_3'] =='o', '15', train['ord_3'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ord = train[['id','ord_0','ord_2','ord_3','month','day','target']]\nord[\"ord_0\"]= ord[\"ord_0\"].astype(float) \nord[\"ord_2\"]= ord[\"ord_2\"].astype(float) \nord[\"ord_3\"]= ord[\"ord_3\"].astype(float)\nord.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ord = ord.interpolate()\nprint(ord.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"correlation_o = ord.corr()\nplt.figure(figsize = (10, 10))\nsns.set(font_scale = 1.5)\nsns.heatmap(correlation_o, annot = True, annot_kws = {'size': 10}, cmap = 'Greens')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}