{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-30T04:38:30.760629Z","iopub.execute_input":"2022-05-30T04:38:30.762258Z","iopub.status.idle":"2022-05-30T04:38:30.767851Z","shell.execute_reply.started":"2022-05-30T04:38:30.762196Z","shell.execute_reply":"2022-05-30T04:38:30.766739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = '/kaggle/input/cat-in-the-dat-ii/'\n\ntrain = pd.read_csv(data_path + 'train.csv', index_col='id')\ntest = pd.read_csv(data_path + 'test.csv', index_col='id')\nsubmission = pd.read_csv(data_path + 'sample_submission.csv', index_col='id')","metadata":{"execution":{"iopub.status.busy":"2022-05-30T04:39:28.183958Z","iopub.execute_input":"2022-05-30T04:39:28.184485Z","iopub.status.idle":"2022-05-30T04:39:34.575031Z","shell.execute_reply.started":"2022-05-30T04:39:28.184447Z","shell.execute_reply":"2022-05-30T04:39:34.574183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape, test.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-30T04:41:22.290327Z","iopub.execute_input":"2022-05-30T04:41:22.290766Z","iopub.status.idle":"2022-05-30T04:41:22.299738Z","shell.execute_reply.started":"2022-05-30T04:41:22.290735Z","shell.execute_reply":"2022-05-30T04:41:22.297936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head().T","metadata":{"execution":{"iopub.status.busy":"2022-05-30T04:39:44.785898Z","iopub.execute_input":"2022-05-30T04:39:44.786410Z","iopub.status.idle":"2022-05-30T04:39:44.805616Z","shell.execute_reply.started":"2022-05-30T04:39:44.786369Z","shell.execute_reply":"2022-05-30T04:39:44.804557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-30T04:40:24.162447Z","iopub.execute_input":"2022-05-30T04:40:24.162884Z","iopub.status.idle":"2022-05-30T04:40:24.196783Z","shell.execute_reply.started":"2022-05-30T04:40:24.162832Z","shell.execute_reply":"2022-05-30T04:40:24.195476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-30T04:42:15.130192Z","iopub.execute_input":"2022-05-30T04:42:15.131517Z","iopub.status.idle":"2022-05-30T04:42:15.142557Z","shell.execute_reply.started":"2022-05-30T04:42:15.131464Z","shell.execute_reply":"2022-05-30T04:42:15.141583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 피처 요약표 만들기","metadata":{}},{"cell_type":"code","source":"def resumetable(df):\n    print(f'데이터셋 형상: {df.shape}')\n    summary = pd.DataFrame(df.dtypes, columns=['데이터 타입'])\n    summary = summary.reset_index()\n    summary = summary.rename(columns={'index': '피처'})\n    summary['결측값 개수'] = df.isnull().sum().values\n    summary['고윳값 개수'] = df.nunique().values\n    summary['첫 번째 값'] = df.loc[0].values\n    summary['두 번째 값'] = df.loc[1].values\n    summary['세 번째 값'] = df.loc[2].values\n    \n    return summary","metadata":{"execution":{"iopub.status.busy":"2022-05-30T04:44:06.932152Z","iopub.execute_input":"2022-05-30T04:44:06.932773Z","iopub.status.idle":"2022-05-30T04:44:06.941094Z","shell.execute_reply.started":"2022-05-30T04:44:06.932724Z","shell.execute_reply":"2022-05-30T04:44:06.940151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_resume_table = resumetable(train)\nfeature_resume_table","metadata":{"execution":{"iopub.status.busy":"2022-05-30T04:46:20.604283Z","iopub.execute_input":"2022-05-30T04:46:20.604911Z","iopub.status.idle":"2022-05-30T04:46:22.787597Z","shell.execute_reply.started":"2022-05-30T04:46:20.604858Z","shell.execute_reply":"2022-05-30T04:46:22.786522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 피처 요약표 해석하기\n\n1. 이진(binary) 피처 : bin_0 ~ bin_4\n2. 명목형(nominal) 피처 : nom_0 ~ nom_9\n3. 순서형(ordinal) 피처 : ord_0 ~ ord_5\n4. 그 외 피처 : day, month, target","metadata":{}},{"cell_type":"code","source":"feature_resume_table[feature_resume_table['피처'].str.startswith('bin')]","metadata":{"execution":{"iopub.status.busy":"2022-05-30T04:46:44.379363Z","iopub.execute_input":"2022-05-30T04:46:44.379840Z","iopub.status.idle":"2022-05-30T04:46:44.398839Z","shell.execute_reply.started":"2022-05-30T04:46:44.379806Z","shell.execute_reply":"2022-05-30T04:46:44.397964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"이진 피처\n\n고윳값이 모두 2개\n\nbin_0 ~ bin_2 는 데이터 타입이 float64  실제값이 0.0, 1.0 으로 구성\n\nbin_3, bin_4 는 object 타입이고 실제값은 T 또는 F (bin_3), Y 또는 N (bin_4)\n\n결측값은 약 18000개씩 존재 \n\n- 최빈값으로 처리할까?","metadata":{}},{"cell_type":"code","source":"feature_resume_table[feature_resume_table['피처'].str.startswith('nom')]","metadata":{"execution":{"iopub.status.busy":"2022-05-30T04:49:26.947091Z","iopub.execute_input":"2022-05-30T04:49:26.947740Z","iopub.status.idle":"2022-05-30T04:49:26.964948Z","shell.execute_reply.started":"2022-05-30T04:49:26.947690Z","shell.execute_reply":"2022-05-30T04:49:26.963963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"명목형 피처\n\n모두 object 타입이고 결측값이 약 18000개 정도씩 있음\n\nnom_0 ~ nom_4 는 고윳값이 6개 이하인데, nom_5 ~ nom_9 는 고윳값이 많음\n\nnom_5 ~ nom_9 는 알 수 없는 값이 입력되어 있음","metadata":{}},{"cell_type":"code","source":"feature_resume_table[feature_resume_table['피처'].str.startswith('ord')]","metadata":{"execution":{"iopub.status.busy":"2022-05-30T05:18:38.560796Z","iopub.execute_input":"2022-05-30T05:18:38.562163Z","iopub.status.idle":"2022-05-30T05:18:38.577025Z","shell.execute_reply.started":"2022-05-30T05:18:38.562115Z","shell.execute_reply":"2022-05-30T05:18:38.575851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###### 순서형 피처\n\nord_0 피처만 float64 타입이고 나머지는 object 타입\n\n결측값은 역시 약 18000개\n\n순서를 파악하기 위해 순서형 피처의 고윳값을 출력해보자\n\n고윳값 개수가 적은 ord_0 ~ ord_5 피처부터 보자","metadata":{}},{"cell_type":"code","source":"for i in range(6):\n    feature = 'ord_' + str(i)\n    print(f'{feature} 고윳값: {train[feature].unique()}')","metadata":{"execution":{"iopub.status.busy":"2022-05-30T04:58:52.264127Z","iopub.execute_input":"2022-05-30T04:58:52.265238Z","iopub.status.idle":"2022-05-30T04:58:52.573499Z","shell.execute_reply.started":"2022-05-30T04:58:52.265196Z","shell.execute_reply":"2022-05-30T04:58:52.572056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"ord_0 는 숫자로 되어있어서 숫자 크기에 순서를 맞춤\n\nord_1 은 'Novice', 'Contributor', 'Expert', 'Master', 'Grandmaster' 순으로 맞춤\n\nord_2 는 'Freezing', 'Cold', 'Warm', 'Hot', 'Boiling Hot', 'Lava Hot' 순으로 맞춤\n\nord_3 ~ ord_5 는 알파벳 순으로 맞춤","metadata":{}},{"cell_type":"code","source":"feature_resume_table[~feature_resume_table['피처'].str.contains('_')]","metadata":{"execution":{"iopub.status.busy":"2022-05-30T05:03:15.757028Z","iopub.execute_input":"2022-05-30T05:03:15.757413Z","iopub.status.idle":"2022-05-30T05:03:15.772522Z","shell.execute_reply.started":"2022-05-30T05:03:15.757384Z","shell.execute_reply":"2022-05-30T05:03:15.771477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('day 고윳값:', train['day'].unique())\nprint('month 고윳값:', train['month'].unique())\nprint('target 고윳값:', train['target'].unique())","metadata":{"execution":{"iopub.status.busy":"2022-05-30T05:04:11.906666Z","iopub.execute_input":"2022-05-30T05:04:11.907317Z","iopub.status.idle":"2022-05-30T05:04:11.932846Z","shell.execute_reply.started":"2022-05-30T05:04:11.907278Z","shell.execute_reply":"2022-05-30T05:04:11.931819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"day 의 고윳값은 7개 이므로 요일을 나타낸다고 짐작할 수 있음\n\nmonth 의 고윳값은 1 ~ 12 이므로 월을 나타냄\n\ntarget 의 고윳값은 0 또는 1로 구성","metadata":{}},{"cell_type":"markdown","source":"## 데이터 시각화","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-05-30T05:05:18.933005Z","iopub.execute_input":"2022-05-30T05:05:18.933849Z","iopub.status.idle":"2022-05-30T05:05:20.321644Z","shell.execute_reply.started":"2022-05-30T05:05:18.933805Z","shell.execute_reply":"2022-05-30T05:05:20.320653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 타깃값 분포","metadata":{}},{"cell_type":"code","source":"def write_percent(ax, total_size):\n    '''도형 객체를 순회하며 막대 상단에 타깃값 비율 표시'''\n    for patch in ax.patches:\n        height = patch.get_height()\n        width = patch.get_width()\n        left_coord = patch.get_x()\n        percent = height / total_size * 100\n        \n        ax.text(x=left_coord + width / 2.0,\n                y=height + total_size * 0.001,\n                s=f'{percent:1.1f}%',\n                ha='center')","metadata":{"execution":{"iopub.status.busy":"2022-05-30T05:08:28.128010Z","iopub.execute_input":"2022-05-30T05:08:28.128394Z","iopub.status.idle":"2022-05-30T05:08:28.135785Z","shell.execute_reply.started":"2022-05-30T05:08:28.128364Z","shell.execute_reply":"2022-05-30T05:08:28.134998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mpl.rc('font', size=15)\nplt.figure(figsize=(7, 6))\n\n# 타깃값 분포 카운트플롯\nax = sns.countplot(x='target', data=train)\nwrite_percent(ax, len(train))\nax.set_title('Target Distribution')","metadata":{"execution":{"iopub.status.busy":"2022-05-30T05:08:29.398739Z","iopub.execute_input":"2022-05-30T05:08:29.399361Z","iopub.status.idle":"2022-05-30T05:08:29.633580Z","shell.execute_reply.started":"2022-05-30T05:08:29.399316Z","shell.execute_reply":"2022-05-30T05:08:29.632309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 분석 결과 : 타깃값 0과 1이 약 8대 2 비율","metadata":{}},{"cell_type":"markdown","source":"### 이진 피처 분포","metadata":{}},{"cell_type":"code","source":"import matplotlib.gridspec as gridspec","metadata":{"execution":{"iopub.status.busy":"2022-05-30T05:09:23.884019Z","iopub.execute_input":"2022-05-30T05:09:23.884500Z","iopub.status.idle":"2022-05-30T05:09:23.890587Z","shell.execute_reply.started":"2022-05-30T05:09:23.884456Z","shell.execute_reply":"2022-05-30T05:09:23.889231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 3행 2열 틀(Figure) 준비\nmpl.rc('font', size=12)\ngrid = gridspec.GridSpec(3, 2)\nplt.figure(figsize=(10, 16))\nplt.subplots_adjust(wspace=0.4, hspace=0.3)\n\n# 서브플롯 그리기\nbin_features = ['bin_' + str(i) for i in range(5)]\n\nfor idx, feature in enumerate(bin_features):\n    ax = plt.subplot(grid[idx])\n    \n    # ax 축에 타깃값 분포 카운트플롯 그리기\n    sns.countplot(x=feature,\n                  data=train,\n                  hue='target',\n                  palette='pastel',\n                  ax=ax)\n    \n    ax.set_title(f'{feature} Distribution by Target')\n    write_percent(ax, len(train))","metadata":{"execution":{"iopub.status.busy":"2022-05-30T05:12:18.649733Z","iopub.execute_input":"2022-05-30T05:12:18.650996Z","iopub.status.idle":"2022-05-30T05:12:21.468432Z","shell.execute_reply.started":"2022-05-30T05:12:18.650941Z","shell.execute_reply":"2022-05-30T05:12:21.467239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 분석 결과 : 고윳값별로 나눠봐도 타깃값 0, 1의 분포가 대체포 4:1 수준임, 특정 타깃값에 치우치지 않았음을 확인할 수 있음","metadata":{}},{"cell_type":"markdown","source":"### 명목형 피처 분포\n\nnom_5 ~ nom_9 피처는 고윳값 개수가 많고 의미를 알 수 없는 문자열이 입력되어 있으니 nom_0 ~ nom_4 피처까지만 시각화하기로 함","metadata":{}},{"cell_type":"code","source":"def get_crosstab(df, feature):\n    crosstab = pd.crosstab(df[feature], df['target'], normalize='index') * 100\n    crosstab = crosstab.reset_index()\n    return crosstab","metadata":{"execution":{"iopub.status.busy":"2022-05-30T05:19:34.453728Z","iopub.execute_input":"2022-05-30T05:19:34.454408Z","iopub.status.idle":"2022-05-30T05:19:34.460787Z","shell.execute_reply.started":"2022-05-30T05:19:34.454367Z","shell.execute_reply":"2022-05-30T05:19:34.459529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_pointplot(ax, feature, crosstab):\n    ax2 = ax.twinx()\n    \n    ax2 = sns.pointplot(x=feature, y=1, data=crosstab,\n                        order=crosstab[feature].values,\n                        color='black',\n                        legend=False,\n                       )\n    ax2.set_ylim(crosstab[1].min() - 5, crosstab[1].max() * 1.1)\n    ax2.set_ylabel('Target 1 Ratio(%)')","metadata":{"execution":{"iopub.status.busy":"2022-05-30T05:26:38.779226Z","iopub.execute_input":"2022-05-30T05:26:38.779647Z","iopub.status.idle":"2022-05-30T05:26:38.787984Z","shell.execute_reply.started":"2022-05-30T05:26:38.779611Z","shell.execute_reply":"2022-05-30T05:26:38.786760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_cat_dist_with_true_ratio(df, features, num_rows, num_cols, size=(15, 20)):\n    plt.figure(figsize=size)\n    grid = gridspec.GridSpec(num_rows, num_cols)\n    plt.subplots_adjust(wspace=0.4, hspace=0.3)\n    \n    for idx, feature in enumerate(features):\n        ax = plt.subplot(grid[idx])\n        crosstab = get_crosstab(df, feature)\n        \n        sns.countplot(x=feature, data=df,\n                      order=crosstab[feature].values,\n                      color='skyblue',\n                      ax=ax)\n        \n        write_percent(ax, len(df))\n        \n        plot_pointplot(ax, feature, crosstab)\n        \n        ax.set_title(f'{feature} Distribution')","metadata":{"execution":{"iopub.status.busy":"2022-05-30T05:26:39.564878Z","iopub.execute_input":"2022-05-30T05:26:39.565354Z","iopub.status.idle":"2022-05-30T05:26:39.575932Z","shell.execute_reply.started":"2022-05-30T05:26:39.565318Z","shell.execute_reply":"2022-05-30T05:26:39.574202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nom_features = ['nom_' + str(i) for i in range(5)]\nplot_cat_dist_with_true_ratio(train, nom_features, num_rows=3, num_cols=2)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T05:26:40.120461Z","iopub.execute_input":"2022-05-30T05:26:40.120956Z","iopub.status.idle":"2022-05-30T05:26:45.113258Z","shell.execute_reply.started":"2022-05-30T05:26:40.120919Z","shell.execute_reply":"2022-05-30T05:26:45.112450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 분석 결과 : nom_0 ~ nom_4 피처는 고윳값별로 타깃값 1 비율이 서로 다름, 고윳값 개수도 적으니 원-핫 인코딩 적용","metadata":{}},{"cell_type":"markdown","source":"### 순서형 피처 분포","metadata":{}},{"cell_type":"code","source":"from pandas.api.types import CategoricalDtype","metadata":{"execution":{"iopub.status.busy":"2022-05-30T05:27:20.980406Z","iopub.execute_input":"2022-05-30T05:27:20.981610Z","iopub.status.idle":"2022-05-30T05:27:20.986245Z","shell.execute_reply.started":"2022-05-30T05:27:20.981558Z","shell.execute_reply":"2022-05-30T05:27:20.985368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ord_1_value = ['Novice', 'Contributor', 'Expert', 'Master', 'Grandmaster']\nord_2_value = ['Freezing', 'Cold', 'Warm', 'Hot', 'Boiling Hot', 'Lava Hot']\n\n# 순서를 지정한 범주형 데이터 타입\nord_1_dtype = CategoricalDtype(categories=ord_1_value, ordered=True)\nord_2_dtype = CategoricalDtype(categories=ord_2_value, ordered=True)\n\n# 데이터 타입 변경\ntrain['ord_1'] = train['ord_1'].astype(ord_1_dtype)\ntrain['ord_2'] = train['ord_2'].astype(ord_2_dtype)","metadata":{"execution":{"iopub.status.busy":"2022-05-30T05:29:31.036427Z","iopub.execute_input":"2022-05-30T05:29:31.037953Z","iopub.status.idle":"2022-05-30T05:29:31.532285Z","shell.execute_reply.started":"2022-05-30T05:29:31.037892Z","shell.execute_reply":"2022-05-30T05:29:31.531150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ord_features = ['ord_' + str(i) for i in range(4)]\nplot_cat_dist_with_true_ratio(train, ord_features, num_rows=2, num_cols=2, size=(15, 12))","metadata":{"execution":{"iopub.status.busy":"2022-05-30T05:29:33.524476Z","iopub.execute_input":"2022-05-30T05:29:33.525146Z","iopub.status.idle":"2022-05-30T05:29:36.339541Z","shell.execute_reply.started":"2022-05-30T05:29:33.525110Z","shell.execute_reply":"2022-05-30T05:29:36.337860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 분석 결과 : ord_0 ~ ord_3 모두 고윳값 순서에 따라 타깃값 1 비율도 비례해서 커짐","metadata":{}},{"cell_type":"code","source":"plot_cat_dist_with_true_ratio(train, ['ord_4', 'ord_5'], num_rows=2, num_cols=1, size=(15, 12))","metadata":{"execution":{"iopub.status.busy":"2022-05-30T05:30:51.828548Z","iopub.execute_input":"2022-05-30T05:30:51.829022Z","iopub.status.idle":"2022-05-30T05:30:57.324173Z","shell.execute_reply.started":"2022-05-30T05:30:51.828987Z","shell.execute_reply":"2022-05-30T05:30:57.322976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 분석 결과 : ord_4, ord_5 모두 고윳값 순서에 따라 타깃값 1 비율이 증가","metadata":{}},{"cell_type":"markdown","source":"### 날짜 피처 분포","metadata":{}},{"cell_type":"code","source":"date_features = ['day', 'month']\nplot_cat_dist_with_true_ratio(train, date_features, num_rows=2, num_cols=1, size=(10, 10))","metadata":{"execution":{"iopub.status.busy":"2022-05-30T05:31:52.997838Z","iopub.execute_input":"2022-05-30T05:31:52.998417Z","iopub.status.idle":"2022-05-30T05:31:54.090157Z","shell.execute_reply.started":"2022-05-30T05:31:52.998379Z","shell.execute_reply":"2022-05-30T05:31:54.089148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 분석 결과 : \n    - day 피처는 1에서 4로 갈수록 타깃값 1 비율이 줄어들고 4에서 7로 갈수록 비율이 증가\n    - month 피처는 1에서 2, 6에서 8, 11에서 12로 갈때 줄어들고 2에서 6, 8에서 11로 갈 때 비율이 증가","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}