{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":74571,"databundleVersionId":8671431,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2024-06-27T08:38:05.239133Z","iopub.execute_input":"2024-06-27T08:38:05.239651Z","iopub.status.idle":"2024-06-27T08:38:05.249611Z","shell.execute_reply.started":"2024-06-27T08:38:05.239604Z","shell.execute_reply":"2024-06-27T08:38:05.248328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 데이터 이해 및 탐색","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-27T08:17:02.043492Z","iopub.execute_input":"2024-06-27T08:17:02.044028Z","iopub.status.idle":"2024-06-27T08:17:02.700635Z","shell.execute_reply.started":"2024-06-27T08:17:02.043995Z","shell.execute_reply":"2024-06-27T08:17:02.699664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/affel-ds2-credit-card/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/affel-ds2-credit-card/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-06-27T08:17:02.702318Z","iopub.execute_input":"2024-06-27T08:17:02.702634Z","iopub.status.idle":"2024-06-27T08:17:04.982798Z","shell.execute_reply.started":"2024-06-27T08:17:02.702605Z","shell.execute_reply":"2024-06-27T08:17:04.981841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2024-06-27T08:17:04.984095Z","iopub.execute_input":"2024-06-27T08:17:04.984439Z","iopub.status.idle":"2024-06-27T08:17:05.007367Z","shell.execute_reply.started":"2024-06-27T08:17:04.984412Z","shell.execute_reply":"2024-06-27T08:17:05.006339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.info()","metadata":{"execution":{"iopub.status.busy":"2024-06-27T08:17:05.009417Z","iopub.execute_input":"2024-06-27T08:17:05.009745Z","iopub.status.idle":"2024-06-27T08:17:05.028223Z","shell.execute_reply.started":"2024-06-27T08:17:05.009718Z","shell.execute_reply":"2024-06-27T08:17:05.027219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"해당 데이터는 결측치가 존재하지 않음","metadata":{}},{"cell_type":"code","source":"x_train = train.drop(['id', 'Time'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T08:17:05.029543Z","iopub.execute_input":"2024-06-27T08:17:05.029871Z","iopub.status.idle":"2024-06-27T08:17:05.059850Z","shell.execute_reply.started":"2024-06-27T08:17:05.029844Z","shell.execute_reply":"2024-06-27T08:17:05.058889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- id과 time은 데이터 생성 관련한 작업용 속성이므로 예측에 도움을 주지 않아 제거\n- Class는 타겟 변수","metadata":{}},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)\nx_train.head(3)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T08:17:05.060973Z","iopub.execute_input":"2024-06-27T08:17:05.061262Z","iopub.status.idle":"2024-06-27T08:17:05.091513Z","shell.execute_reply.started":"2024-06-27T08:17:05.061238Z","shell.execute_reply":"2024-06-27T08:17:05.090626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 데이터 전처리 과정\n## 관련성 없는 피처 검색","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(10,10))\ncorr = x_train.corr()\nsns.heatmap(corr, cmap='RdBu')","metadata":{"execution":{"iopub.status.busy":"2024-06-27T08:17:05.093004Z","iopub.execute_input":"2024-06-27T08:17:05.093321Z","iopub.status.idle":"2024-06-27T08:17:06.239245Z","shell.execute_reply.started":"2024-06-27T08:17:05.093295Z","shell.execute_reply":"2024-06-27T08:17:06.237941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"abs_corr = corr['Class'].abs()\nabs_corr = abs_corr[abs_corr <=0.05].index.tolist()\nabs_corr","metadata":{"execution":{"iopub.status.busy":"2024-06-27T08:17:06.240882Z","iopub.execute_input":"2024-06-27T08:17:06.241306Z","iopub.status.idle":"2024-06-27T08:17:06.249960Z","shell.execute_reply.started":"2024-06-27T08:17:06.241266Z","shell.execute_reply":"2024-06-27T08:17:06.248737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"상관관계가 0.05보다 낮은 피처를 저장 후 해당 피처에서 이상치 제거를 진행","metadata":{}},{"cell_type":"markdown","source":"## 이상치 제거","metadata":{}},{"cell_type":"code","source":"x_train.describe()","metadata":{"execution":{"iopub.status.busy":"2024-06-27T08:17:06.252532Z","iopub.execute_input":"2024-06-27T08:17:06.252907Z","iopub.status.idle":"2024-06-27T08:17:06.544328Z","shell.execute_reply.started":"2024-06-27T08:17:06.252879Z","shell.execute_reply":"2024-06-27T08:17:06.543303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,5))\n\nsns.boxplot(x_train.drop(['Amount', 'Class'], axis=1))","metadata":{"execution":{"iopub.status.busy":"2024-06-27T08:17:06.545897Z","iopub.execute_input":"2024-06-27T08:17:06.546316Z","iopub.status.idle":"2024-06-27T08:17:08.231935Z","shell.execute_reply.started":"2024-06-27T08:17:06.546279Z","shell.execute_reply":"2024-06-27T08:17:08.230712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x_train['Amount'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def del_outliers(train, abs_corr, weight=3):\n    \n    for i in abs_corr:\n        train25 = train[i].quantile(0.25)\n        train75 = train[i].quantile(.75)\n        \n        iqr = train75 - train25\n        \n        out_max = train75 + weight * iqr\n        out_min = train25 - weight * iqr\n        \n        train = train[(train[i] <= out_max) & (train[i] >= out_min)]\n        \n    return train","metadata":{"execution":{"iopub.status.busy":"2024-06-27T08:17:42.389954Z","iopub.execute_input":"2024-06-27T08:17:42.390369Z","iopub.status.idle":"2024-06-27T08:17:42.396892Z","shell.execute_reply.started":"2024-06-27T08:17:42.390338Z","shell.execute_reply":"2024-06-27T08:17:42.395725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"이상치 제거 함수\n- 이상치 판단 공식을 통해 특정 값 이상이거나 이하일 경우 이상치로 판단하고 제거\n- 상관계수가 낮은 피처들을 찾고 해당 피처들에서 이상치를 제거\n- 공식은 1.5를 곱하지만 1.5를 곱할 경우 데이터의 2/3가 삭제가 되어서 3으로 곱함","metadata":{}},{"cell_type":"code","source":"x_train = del_outliers(x_train, abs_corr)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T08:17:47.677140Z","iopub.execute_input":"2024-06-27T08:17:47.677608Z","iopub.status.idle":"2024-06-27T08:17:48.001698Z","shell.execute_reply.started":"2024-06-27T08:17:47.677574Z","shell.execute_reply":"2024-06-27T08:17:48.000616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,5))\n\nsns.boxplot(x_train.drop(['Amount', 'Class'], axis=1))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x_train['Amount'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"인코딩이 필요한가? </br>\n해당 데이터 셋은 문자열이 존재하지 않으므로 인코딩은 필요가 없음 </br>\n</br>\n그렇다면 스케일링은 필요한가? </br>\nV1 ~ V28이 무슨 피처인지는 정확히 알 수 없으나 </br>\n트리기반 모델을 사용할 경우 영향이 없기 때문에 스케일링을 할 필요는 없음</br>\n아닐 경우 금액을 나타내는 Amount와는 단위가 다른 것은 알 수 있으므로 스케일링 작업이 필요함","metadata":{}},{"cell_type":"markdown","source":"# 모델 선택 및 비교","metadata":{}},{"cell_type":"code","source":"y_train = x_train['Class']\nx_train = x_train.drop(['Class'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T08:17:50.772468Z","iopub.execute_input":"2024-06-27T08:17:50.772897Z","iopub.status.idle":"2024-06-27T08:17:50.791082Z","shell.execute_reply.started":"2024-06-27T08:17:50.772863Z","shell.execute_reply":"2024-06-27T08:17:50.789771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## SMOTE 오버 샘플링 적용","metadata":{}},{"cell_type":"code","source":"from imblearn.over_sampling import SMOTE\n\nX_tr, X_val, y_tr, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=0)\n\nsmote = SMOTE(random_state=0)\n\nX_train, y_train = smote.fit_resample(X_tr, y_tr)\nprint(f\"적용 전 : {X_tr.shape}, {y_tr.shape}\")\nprint(f\"적용 전 분포 : \\n{pd.Series(y_tr).value_counts()}\")\nprint(f\"적용 후 : {X_train.shape}, {y_train.shape}\")\nprint(f\"적용 후 분포 : \\n{pd.Series(y_train).value_counts()}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-27T08:17:53.836764Z","iopub.execute_input":"2024-06-27T08:17:53.837188Z","iopub.status.idle":"2024-06-27T08:17:54.202771Z","shell.execute_reply.started":"2024-06-27T08:17:53.837156Z","shell.execute_reply":"2024-06-27T08:17:54.201635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 랜덤 포레스트\n결정 트리이므로 스케일링과 정규화 같은 전처리 작업이 필요없음 </br>\n하지만 과적합으로 정확도가 떨어짐","metadata":{}},{"cell_type":"code","source":"# x_test = test.drop(['id', 'Time'], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.ensemble import RandomForestClassifier\n\n# rf_clf = RandomForestClassifier(random_state=0, max_depth=8)\n\n# rf_clf.fit(X_tr, y_tr)\n\n# pred = dt_clf.predict(x_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission = pd.DataFrame({'id': test['id'], 'Class': pred})\n# submission.to_csv('submission_rf.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### public score = 0.56789","metadata":{}},{"cell_type":"markdown","source":"# GBM\n약한 학습기를 순차적으로 학습-예측하면서 잘못 예측한 데이터에 가중치를 부여를 하며 학습을 개선","metadata":{}},{"cell_type":"code","source":"# from sklearn.ensemble import GradientBoostingClassifier\n\n# X_tr, X_val, y_tr, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=0)\n\n# x_test = test.drop(['id', 'Time'], axis=1)\n\n# gb_clf = GradientBoostingClassifier(random_state=0)\n# gb_clf.fit(X_tr, y_tr)\n\n# gb_pred = gb_clf.predict(x_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission = pd.DataFrame({'id': test['id'], 'Class': gb_pred})\n# submission.to_csv('submission_gbm.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### public score = 0.50358","metadata":{}},{"cell_type":"markdown","source":"# XGBoost\nGBM의 느린 수행 시간 및 과적합 규제 부재 등 문제를 해결","metadata":{}},{"cell_type":"code","source":"# from xgboost import XGBClassifier","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_tr, X_val, y_tr, y_val = train_test_split(x_train, y_train, test_size=0.2, random_state=0)\n\n# x_test = test.drop(['id', 'Time'], axis=1)\n\n# params = {\n#          'n_estimators' : 500,\n#          'max_depth' : 5,\n#          'learning_rate' : 0.05,\n#          'min_child_weight' : 5,\n#          'colsample_bytree' : 0.5,\n#          }\n\n# eval_list = [(X_tr, y_tr), (X_val, y_val)]\n\n# xgb_clf = XGBClassifier(**params)\n\n# xgb_clf.fit(X_tr, y_tr, early_stopping_rounds=50, eval_metric=\"auc\", eval_set=eval_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# x_test = test.drop(['id', 'Time'], axis=1)\n\n# xgb_pred = xgb_clf.predict(x_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission = pd.DataFrame({'id': test['id'], 'Class': xgb_pred})\n# submission.to_csv('submission_xgb.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### public score = 0.500000","metadata":{}},{"cell_type":"markdown","source":"# LightGBM\nXGBoost보다 학습시간이 적으며 메모리 사용량도 적음 </br>\n카테고리형 피처의 자동 변환과 최적 분할","metadata":{}},{"cell_type":"code","source":"from lightgbm import LGBMClassifier\n\nx_test = test.drop(['id', 'Time'], axis=1)\n\nparams = {\n        'n_estimators' : 2000,\n#         'num_leaves' : 64,\n        'max_depth' : 6, \n        'learning_rate' : 0.02,\n        'random_state' : 0,\n        'class_weight' : 'balanced',\n        'verbose' : -1\n        }\n\n# eval_list = [(X_train, y_train), (X_val, y_val)]\n\nlgb_clf = LGBMClassifier(**params)\n\n# lgb_clf.fit(X_train, y_train, eval_metric=\"auc\", eval_set=eval_list)\nlgb_clf.fit(X_train, y_train)\n\nlgb_pred = lgb_clf.predict(x_test)\n\nsubmission = pd.DataFrame({'id': test['id'], 'Class': lgb_pred})\nsubmission.to_csv('submission_lgb7.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T08:34:51.381133Z","iopub.execute_input":"2024-06-27T08:34:51.381519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_pred = lgb_clf.predict(X_val)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T08:24:50.723958Z","iopub.execute_input":"2024-06-27T08:24:50.724964Z","iopub.status.idle":"2024-06-27T08:24:56.722225Z","shell.execute_reply.started":"2024-06-27T08:24:50.724929Z","shell.execute_reply":"2024-06-27T08:24:56.721340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report, accuracy_score, precision_score, recall_score, f1_score, roc_auc_score\n\nprint(classification_report(y_val, lgb_pred))\nprint(\"Accuracy:\", accuracy_score(y_val, lgb_pred))\nprint(\"Precision:\", precision_score(y_val, lgb_pred))\nprint(\"Recall:\", recall_score(y_val, lgb_pred))\nprint(\"F1 Score:\", f1_score(y_val, lgb_pred))\nprint(\"ROC-AUC:\", roc_auc_score(y_val, lgb_pred))","metadata":{"execution":{"iopub.status.busy":"2024-06-27T08:24:56.724084Z","iopub.execute_input":"2024-06-27T08:24:56.724476Z","iopub.status.idle":"2024-06-27T08:24:56.805764Z","shell.execute_reply.started":"2024-06-27T08:24:56.724440Z","shell.execute_reply":"2024-06-27T08:24:56.804670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 하이퍼 파라미터 튜닝\n진행 중 시간 부족으로 인해 최적의 파라미터를 찾지는 못하였음","metadata":{}},{"cell_type":"code","source":"from hyperopt import hp, fmin, tpe, Trials\nfrom sklearn.model_selection import StratifiedKFold\n\nlgbm_search_space = {\n                    'max_depth' : hp.quniform('max_depth', 3, 12, 1),\n                    'min_child_samples' : hp.quniform('min_child_samples', 60, 100, 1),\n                    'subsample' : hp.uniform('subsample', 0.7, 1),\n                    'learning_rate' : hp.uniform('learning_rate', 0.01, 0.2)\n                    }\n\ndef objective_func(search_space):\n    lgbm_clf = LGBMClassifier(n_estimators=100,\n                       num_leaves=32,\n                       max_depth=int(search_space['max_depth']),\n                       min_child_weight=int(search_space['min_child_samples']),\n                       colsample_bytree=search_space['subsample'],\n                       learning_rate=search_space['learning_rate'],\n                       verbose = -1\n                     )\n    \n    roc_auc_list = []\n    \n    kf = StratifiedKFold(n_splits=3)\n    \n    for tr_index, val_index in kf.split(X_train, y_train):\n        X_tr, y_tr = X_train.iloc[tr_index], y_train.iloc[tr_index]\n        X_va, y_va = X_train.iloc[val_index], y_train.iloc[val_index]\n        \n        lgbm_clf.fit(X_tr, y_tr, eval_metric=\"auc\", eval_set=[(X_tr, y_tr), (X_va, y_va)])\n        \n        score = roc_auc_score(y_va, lgbm_clf.predict_proba(X_va)[:, -1])\n        roc_auc_list.append(score)\n        \n    return -1*np.mean(roc_auc_list)\n\ntrials = Trials()\n\nbest = fmin(fn=objective_func,\n           space=lgbm_search_space,\n           algo=tpe.suggest,\n           max_evals=50,\n           trials=trials,\n           rstate=np.random.default_rng(seed=30))\n\nprint(best)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T08:38:32.865074Z","iopub.execute_input":"2024-06-27T08:38:32.865479Z"},"trusted":true},"execution_count":null,"outputs":[]}]}