{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-26T00:06:01.91346Z","iopub.execute_input":"2023-02-26T00:06:01.913913Z","iopub.status.idle":"2023-02-26T00:06:01.924264Z","shell.execute_reply.started":"2023-02-26T00:06:01.913874Z","shell.execute_reply":"2023-02-26T00:06:01.922624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = '/kaggle/input/cat-in-the-dat-ii/'\n\ntrain = pd.read_csv(data_path + 'train.csv', index_col=\"id\")\ntest = pd.read_csv(data_path+'test.csv', index_col=\"id\")\nsubmission = pd.read_csv(data_path+'sample_submission.csv', index_col=\"id\")","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:06:06.057574Z","iopub.execute_input":"2023-02-26T00:06:06.058001Z","iopub.status.idle":"2023-02-26T00:06:12.757187Z","shell.execute_reply.started":"2023-02-26T00:06:06.057965Z","shell.execute_reply":"2023-02-26T00:06:12.755896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**성능 개선 전에 결측치 제거**","metadata":{}},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:06:13.810279Z","iopub.execute_input":"2023-02-26T00:06:13.810984Z","iopub.status.idle":"2023-02-26T00:06:14.342773Z","shell.execute_reply.started":"2023-02-26T00:06:13.810924Z","shell.execute_reply":"2023-02-26T00:06:14.341713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#결측치가 포함된 row 제거\ntrain.dropna(axis='index', how=\"any\", inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:06:18.309103Z","iopub.execute_input":"2023-02-26T00:06:18.309568Z","iopub.status.idle":"2023-02-26T00:06:18.978725Z","shell.execute_reply.started":"2023-02-26T00:06:18.309526Z","shell.execute_reply":"2023-02-26T00:06:18.977319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:06:21.354105Z","iopub.execute_input":"2023-02-26T00:06:21.355391Z","iopub.status.idle":"2023-02-26T00:06:21.626904Z","shell.execute_reply.started":"2023-02-26T00:06:21.355326Z","shell.execute_reply":"2023-02-26T00:06:21.625448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**성능개선: 각 피쳐의 특성에 맞는 인코딩(baseline에서는 원핫인코딩으로 전부 돌렸음)**","metadata":{}},{"cell_type":"code","source":"# target값인 y를 나중에 떼어놓을 것을 대비하여 train data 길이 알아놓기\nnum_train = len(train)\nall_data = pd.concat([train, test])\nall_data.head(20)","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:06:32.805452Z","iopub.execute_input":"2023-02-26T00:06:32.806366Z","iopub.status.idle":"2023-02-26T00:06:33.26416Z","shell.execute_reply.started":"2023-02-26T00:06:32.806314Z","shell.execute_reply":"2023-02-26T00:06:33.263158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y=all_data['target'][:num_train]\nall_data = all_data.drop('target', axis=1)\nprint(len(y))","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:06:41.143951Z","iopub.execute_input":"2023-02-26T00:06:41.144358Z","iopub.status.idle":"2023-02-26T00:06:41.827303Z","shell.execute_reply.started":"2023-02-26T00:06:41.144324Z","shell.execute_reply":"2023-02-26T00:06:41.825879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#이진피쳐를 0과 1로 인코딩하기\n#bin_0, 1, 2는 이미 0과 1로 이루어져 있어서 인코딩 필요 없음\n#bin_3은 True, False로, bin_4는 Yes, No로 이뤄져 있음\nall_data['bin_3'] = all_data['bin_3'].map({'F':0, 'T': 1})\nall_data['bin_4'] = all_data['bin_4'].map({'N':0, 'Y':1})","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:06:44.107736Z","iopub.execute_input":"2023-02-26T00:06:44.10812Z","iopub.status.idle":"2023-02-26T00:06:44.620485Z","shell.execute_reply.started":"2023-02-26T00:06:44.108087Z","shell.execute_reply":"2023-02-26T00:06:44.619477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#순서형 피쳐 인코딩 \n#ord_0은 1, 2, 3으로 이뤄져 있어서 인코딩 필요 없음\n#ord_1, 2 피쳐는 순서를 정해서 인코딩\n\nall_data['ord_1'] = all_data['ord_1'].map({'Novice':0, 'Contributor':1, 'Expert':2, 'Master':3, 'Grandmaster':4})\nall_data['ord_2'] = all_data['ord_2'].map({'Freezing':0, 'Cold':1, 'Warm':2, 'Hot':3, 'Boiling Hot':4, 'Lava Hot':5})","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:06:46.751249Z","iopub.execute_input":"2023-02-26T00:06:46.751668Z","iopub.status.idle":"2023-02-26T00:06:47.237389Z","shell.execute_reply.started":"2023-02-26T00:06:46.751634Z","shell.execute_reply":"2023-02-26T00:06:47.235828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ord_3, 4, 5 피쳐는 알파벳 순서로 인코딩\nfrom sklearn.preprocessing import OrdinalEncoder\n\nord_345 = ['ord_3','ord_4', 'ord_5']\n\nord_encoder = OrdinalEncoder()\nall_data[ord_345] = ord_encoder.fit_transform(all_data[ord_345])\n","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:07:20.34054Z","iopub.execute_input":"2023-02-26T00:07:20.340987Z","iopub.status.idle":"2023-02-26T00:07:21.978906Z","shell.execute_reply.started":"2023-02-26T00:07:20.340948Z","shell.execute_reply":"2023-02-26T00:07:21.977625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for feature, categories in zip(ord_345, ord_encoder.categories_):\n    print(feature)\n    print(categories)","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:07:25.4345Z","iopub.execute_input":"2023-02-26T00:07:25.435152Z","iopub.status.idle":"2023-02-26T00:07:25.442929Z","shell.execute_reply.started":"2023-02-26T00:07:25.435097Z","shell.execute_reply":"2023-02-26T00:07:25.441839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data[ord_345]","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:07:32.846043Z","iopub.execute_input":"2023-02-26T00:07:32.846541Z","iopub.status.idle":"2023-02-26T00:07:32.924859Z","shell.execute_reply.started":"2023-02-26T00:07:32.846481Z","shell.execute_reply":"2023-02-26T00:07:32.923442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#명목형 피쳐 인코딩\n#명목형 피쳐에서는 순서가 큰 의미 없음 -> 원핫 인코딩 적용\nfrom sklearn.preprocessing import OneHotEncoder\n\nnoms = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4', 'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']\nonehot_encoder = OneHotEncoder()\norder_data_encoded = onehot_encoder.fit_transform(all_data[noms])\n\norder_data_encoded","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:08:04.413964Z","iopub.execute_input":"2023-02-26T00:08:04.414391Z","iopub.status.idle":"2023-02-26T00:08:06.982884Z","shell.execute_reply.started":"2023-02-26T00:08:04.414352Z","shell.execute_reply":"2023-02-26T00:08:06.981472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#all_data에서 기존의 명목형 데이터 삭제하기. 추후에 원핫인코딩한 명목형 피쳐를 붙일 것\nall_data.drop(noms, axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:08:22.581399Z","iopub.execute_input":"2023-02-26T00:08:22.581858Z","iopub.status.idle":"2023-02-26T00:08:22.643182Z","shell.execute_reply.started":"2023-02-26T00:08:22.58182Z","shell.execute_reply":"2023-02-26T00:08:22.641606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:08:32.682285Z","iopub.execute_input":"2023-02-26T00:08:32.68272Z","iopub.status.idle":"2023-02-26T00:08:32.709835Z","shell.execute_reply.started":"2023-02-26T00:08:32.682682Z","shell.execute_reply":"2023-02-26T00:08:32.708444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#날짜형 데이터 인코딩\n#날짜 피쳐는 숫자의 크고 작음으로 보이지 않기 위해 원핫인코딩\n#기존의 날짜 피쳐 버리고 인코딩해둔 피쳐를 나중에 붙일 것\n\ndate=['day', 'month']\n\ndate_encoded = onehot_encoder.fit_transform(all_data[date])\n\nall_data.drop(date, axis=1, inplace=True)\ndate_encoded","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:09:14.570863Z","iopub.execute_input":"2023-02-26T00:09:14.571307Z","iopub.status.idle":"2023-02-26T00:09:14.854056Z","shell.execute_reply.started":"2023-02-26T00:09:14.571266Z","shell.execute_reply":"2023-02-26T00:09:14.852807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**성능개선: 피쳐스케일**","metadata":{}},{"cell_type":"code","source":"#이진분류, 명목피쳐, 날짜피쳐는 0, 1로 인코딩함\n#순서형 피쳐만 데이터 범위가 다름. 순서형 피쳐값의 범위가 0~1 사이가 되도록 스케일링하기\n\nfrom sklearn.preprocessing import MinMaxScaler\n\nord = ['ord_' + str(i) for i in range(6)]\n\nmin_max_scaler = MinMaxScaler()\nall_data[ord] = min_max_scaler.fit_transform(all_data[ord])","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:09:38.384418Z","iopub.execute_input":"2023-02-26T00:09:38.384877Z","iopub.status.idle":"2023-02-26T00:09:38.468292Z","shell.execute_reply.started":"2023-02-26T00:09:38.384837Z","shell.execute_reply":"2023-02-26T00:09:38.466969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:10:07.113026Z","iopub.execute_input":"2023-02-26T00:10:07.113477Z","iopub.status.idle":"2023-02-26T00:10:07.138405Z","shell.execute_reply.started":"2023-02-26T00:10:07.113441Z","shell.execute_reply":"2023-02-26T00:10:07.136572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#명목형, 날짜형 데이터 붙이기\n#명목형, 날짜 데이터는 원핫인코딩을 해서 CSR 형식임. 데이터프레임형식인 all_data를 CSR 형식으로 바꿔서 합치자\n\nfrom scipy import sparse\n\nall_data_spars = sparse.hstack([sparse.csr_matrix(all_data), order_data_encoded, date_encoded], format='csr')","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:10:46.23064Z","iopub.execute_input":"2023-02-26T00:10:46.231066Z","iopub.status.idle":"2023-02-26T00:10:47.132548Z","shell.execute_reply.started":"2023-02-26T00:10:46.231031Z","shell.execute_reply":"2023-02-26T00:10:47.13127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data_spars","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:10:50.448965Z","iopub.execute_input":"2023-02-26T00:10:50.449355Z","iopub.status.idle":"2023-02-26T00:10:50.456697Z","shell.execute_reply.started":"2023-02-26T00:10:50.449322Z","shell.execute_reply":"2023-02-26T00:10:50.455043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**성능개선: 데이터 나누기(훈련용, 테스트용)**","metadata":{}},{"cell_type":"code","source":"X_train = all_data_spars[:num_train]\nX_test = all_data_spars[num_train:]","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:11:28.076504Z","iopub.execute_input":"2023-02-26T00:11:28.077979Z","iopub.status.idle":"2023-02-26T00:11:28.313464Z","shell.execute_reply.started":"2023-02-26T00:11:28.07791Z","shell.execute_reply":"2023-02-26T00:11:28.311845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_train.shape, X_test.shape)","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:12:25.014665Z","iopub.execute_input":"2023-02-26T00:12:25.015234Z","iopub.status.idle":"2023-02-26T00:12:25.021961Z","shell.execute_reply.started":"2023-02-26T00:12:25.01518Z","shell.execute_reply":"2023-02-26T00:12:25.020873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(y)","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:13:20.233698Z","iopub.execute_input":"2023-02-26T00:13:20.234111Z","iopub.status.idle":"2023-02-26T00:13:20.240499Z","shell.execute_reply.started":"2023-02-26T00:13:20.234078Z","shell.execute_reply":"2023-02-26T00:13:20.239516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**성능개선: 하이퍼파라미터 최적화**","metadata":{}},{"cell_type":"code","source":"%%time\n\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.linear_model import LogisticRegression\n\nlogistic_model = LogisticRegression()\n\nlogistic_params = {'C': [0.1, 0.125, 0.2], 'max_iter':[800, 900, 1000],\n                  'solver':['liblinear'], 'random_state':[42]}\n\ngridsearch_logistic_model = GridSearchCV(estimator = logistic_model, param_grid = logistic_params, scoring='roc_auc', cv=5)\ngridsearch_logistic_model.fit(X_train, y)","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:13:34.387772Z","iopub.execute_input":"2023-02-26T00:13:34.388208Z","iopub.status.idle":"2023-02-26T00:19:44.81807Z","shell.execute_reply.started":"2023-02-26T00:13:34.38817Z","shell.execute_reply":"2023-02-26T00:19:44.816851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('최적의 파라미터: ', gridsearch_logistic_model.best_params_)\nprint(\"최적의 estimator:\", gridsearch_logistic_model.best_estimator_)","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:20:11.917375Z","iopub.execute_input":"2023-02-26T00:20:11.917781Z","iopub.status.idle":"2023-02-26T00:20:11.924588Z","shell.execute_reply.started":"2023-02-26T00:20:11.917745Z","shell.execute_reply":"2023-02-26T00:20:11.923607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**성능개선: 하이퍼파라미터를 적용한 모델 성능 검증**","metadata":{}},{"cell_type":"code","source":"y_preds = gridsearch_logistic_model.best_estimator_.predict_proba(X_train)[:, 1]","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:20:16.27008Z","iopub.execute_input":"2023-02-26T00:20:16.270564Z","iopub.status.idle":"2023-02-26T00:20:16.296252Z","shell.execute_reply.started":"2023-02-26T00:20:16.27052Z","shell.execute_reply":"2023-02-26T00:20:16.294574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(y_preds)","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:20:34.829315Z","iopub.execute_input":"2023-02-26T00:20:34.82977Z","iopub.status.idle":"2023-02-26T00:20:34.836721Z","shell.execute_reply.started":"2023-02-26T00:20:34.829727Z","shell.execute_reply":"2023-02-26T00:20:34.835645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\n\nscore = roc_auc_score(y, y_preds)\nprint(f'검증데이터 ROC AUC: {score:.4f}')","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:20:42.591868Z","iopub.execute_input":"2023-02-26T00:20:42.592375Z","iopub.status.idle":"2023-02-26T00:20:42.72011Z","shell.execute_reply.started":"2023-02-26T00:20:42.592336Z","shell.execute_reply":"2023-02-26T00:20:42.718765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**결과 제출**","metadata":{}},{"cell_type":"code","source":"X_test","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:21:16.90876Z","iopub.execute_input":"2023-02-26T00:21:16.909238Z","iopub.status.idle":"2023-02-26T00:21:16.917702Z","shell.execute_reply.started":"2023-02-26T00:21:16.909144Z","shell.execute_reply":"2023-02-26T00:21:16.916229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = gridsearch_logistic_model.best_estimator_.predict_proba(X_test)[:, 1]\n\n# submission['target'] = y_pred","metadata":{"execution":{"iopub.status.busy":"2023-02-26T00:20:58.07268Z","iopub.execute_input":"2023-02-26T00:20:58.073094Z","iopub.status.idle":"2023-02-26T00:20:58.117748Z","shell.execute_reply.started":"2023-02-26T00:20:58.073059Z","shell.execute_reply":"2023-02-26T00:20:58.115903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}