{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"papermill":{"default_parameters":{},"duration":5232.034539,"end_time":"2022-11-10T16:10:38.379903","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2022-11-10T14:43:26.345364","version":"2.3.4"},"colab":{"provenance":[]},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":5357,"databundleVersionId":38056,"sourceType":"competition"},{"sourceId":6978076,"sourceType":"datasetVersion","datasetId":4009850}],"dockerImageVersionId":30588,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 불량 검출 문제를 풀기 위한 접근 방법\n\n1. 지도학습 문제로 정의 : binary classification (정상 / 불량)\n\n> 두 개의 클래스 사이의 큰 불균형이 존재하기 때문에 해결해야함.\n\n\n2. 비지도학습 문제로 정의 : clustering (2가지로 자동 분류)\n\n> K-means보단, DBSCAN 같은 모델이 주로 사용됨.\n\n\n3. Anomaly Detection 문제로 정의 : 불량을 outlier로 정의해서 대부분의 데이터(majority)와 나머지를 찾는 문제로 정의.\n\n> 다양한 anomaly detection 모델을 사용함.","metadata":{"id":"e34fe8bf","papermill":{"duration":0.015043,"end_time":"2022-11-10T14:43:35.604553","exception":false,"start_time":"2022-11-10T14:43:35.589510","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## 1. 라이브러리, 데이터 불러오기","metadata":{"id":"222f8ad3","papermill":{"duration":0.014616,"end_time":"2022-11-10T14:43:35.660710","exception":false,"start_time":"2022-11-10T14:43:35.646094","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# !pip install optuna torchinfo torchmetrics","metadata":{"id":"srSiW1GUjrxY","outputId":"4b4efcd7-475a-4ca4-98b5-176e48b05cf1","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 데이터분석 4종 세트\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# 모델들, 성능 평가\n# (저는 일반적으로 정형데이터로 머신러닝 분석할 때는 이 2개 모델은 그냥 돌려봅니다. 특히 RF가 테스트하기 좋습니다.)\nfrom lightgbm.sklearn import LGBMClassifier\nfrom imblearn.over_sampling import SMOTE # ML - KNN Graph\n# KFold(CV), partial : optuna를 사용하기 위함\nfrom sklearn.model_selection import KFold\nfrom functools import partial\n\n# hyper-parameter tuning을 위한 라이브러리, optuna\nimport optuna","metadata":{"id":"5ba77e23","papermill":{"duration":3.342311,"end_time":"2022-11-10T14:43:39.017482","exception":false,"start_time":"2022-11-10T14:43:35.675171","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T06:31:14.274628Z","iopub.execute_input":"2023-11-16T06:31:14.274935Z","iopub.status.idle":"2023-11-16T06:31:18.070496Z","shell.execute_reply.started":"2023-11-16T06:31:14.274910Z","shell.execute_reply":"2023-11-16T06:31:18.069451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nimport os\nimport gc\nimport time\nimport random\nimport string\n\nimport copy\nfrom copy import deepcopy\n\nimport torchmetrics\nfrom torchmetrics.classification import BinaryF1Score\n\nfrom sklearn.model_selection import train_test_split\n\n## Pytorch Import\nimport torch\nimport torch.nn as nn\n\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\n\n# Utils\nfrom tqdm.auto import tqdm, trange\n\n# Suppress warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2023-11-16T06:31:18.075881Z","iopub.execute_input":"2023-11-16T06:31:18.076174Z","iopub.status.idle":"2023-11-16T06:31:22.740838Z","shell.execute_reply.started":"2023-11-16T06:31:18.076148Z","shell.execute_reply":"2023-11-16T06:31:22.739612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nseed_everything(42) # Seed 고정","metadata":{"execution":{"iopub.status.busy":"2023-11-16T06:31:22.742069Z","iopub.execute_input":"2023-11-16T06:31:22.742722Z","iopub.status.idle":"2023-11-16T06:31:22.751681Z","shell.execute_reply.started":"2023-11-16T06:31:22.742690Z","shell.execute_reply":"2023-11-16T06:31:22.750879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# flag setting\nsampling = 'hybrid' ## 'under' or 'over' or 'hybrid'","metadata":{"id":"3baf9b7e","papermill":{"duration":0.022911,"end_time":"2022-11-10T14:43:39.055730","exception":false,"start_time":"2022-11-10T14:43:39.032819","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T06:31:22.754092Z","iopub.execute_input":"2023-11-16T06:31:22.754466Z","iopub.status.idle":"2023-11-16T06:31:22.758933Z","shell.execute_reply.started":"2023-11-16T06:31:22.754429Z","shell.execute_reply":"2023-11-16T06:31:22.758075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 데이터를 불러옵니다.\nbase_path = \"/kaggle/input/bosch-production-line-performance/\"\ntrain_cat_part = pd.read_csv(base_path + 'train_categorical.csv.zip',\n                            nrows=1000)\ntrain_num_part = pd.read_csv(base_path + 'train_numeric.csv.zip',\n                            nrows=1000)\ntrain_date_part = pd.read_csv(base_path + 'train_date.csv.zip',\n                             nrows=1000)\n\nprint(train_cat_part.shape, train_num_part.shape, train_date_part.shape)","metadata":{"id":"ff89465f","papermill":{"duration":0.427331,"end_time":"2022-11-10T14:43:39.497080","exception":false,"start_time":"2022-11-10T14:43:39.069749","status":"completed"},"tags":[],"outputId":"961a25ae-296b-4b98-ca34-d10af5ca4a46","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(1183747, 2141+970+1157)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = pd.read_csv(base_path + 'train_numeric.csv.zip', usecols=['Id', 'Response'])\ntemp","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp.Response.value_counts() # 정상 : 불량 = 1176868 : 6879 = 171 : 1 ---->(hybrid) 20000 : 20000 // 100000 : 20000 (5:1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"categorical data는 EDA 결과 분석에 큰 도움이 없어, 사용하지 않겠습니다.","metadata":{"papermill":{"duration":0.013428,"end_time":"2022-11-10T14:43:39.525135","exception":false,"start_time":"2022-11-10T14:43:39.511707","status":"completed"},"tags":[],"id":"6183159f"}},{"cell_type":"code","source":"# total = train_cat_part.count() * 0\n# N = 0\n\n# for chunk in pd.read_csv('../input/bosch-production-line-performance/train_categorical.csv.zip',\n#                         chunksize=100000):  # chunksize가 클수록, 속도는 빨라지고 메모리 사용량이 늘어납니다.\n\n#     total = total + chunk.isnull().sum()  # 전체 데이터의 column별 결측치 개수\n#     N = N + len(chunk)  # 전체 데이터의 row 수\n\n# display(total)\n# print(N)\n# usecols = total.index[(total / N) < 0.5] # train_categorical.csv에 있는 결측치가 50% 미만인 column들.\n# usecols","metadata":{"papermill":{"duration":0.023711,"end_time":"2022-11-10T14:43:39.562587","exception":false,"start_time":"2022-11-10T14:43:39.538876","status":"completed"},"tags":[],"id":"14b8b9ed","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_cat = pd.read_csv('../input/bosch-production-line-performance/train_categorical.csv.zip',\n#                        usecols=usecols)\n# train_cat","metadata":{"papermill":{"duration":0.022751,"end_time":"2022-11-10T14:43:39.599204","exception":false,"start_time":"2022-11-10T14:43:39.576453","status":"completed"},"tags":[],"id":"9773eef8","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"train_date.csv에서는 파생 변수를 만들어서 사용합니다.\n\n> 1. 각 station별로 측정 시간이 모두 있거나, 없거나의 케이스만 존재한다.\n>> 데이터 중에서 station별로 하나의 column만 뽑아서 체크해도 괜찮다.\n\n> 2. 각 station별로 시간이 모두 같지는 않다. 중간에 측정시간이 바뀌는 케이스도 있다.\n>> 데이터 별로 min, max값을 계산하면 된다.\n\n- start_station : 해당 product가 처음으로 공정을 거친 station_name\n\n- end_station : 해당 product가 마지막으로 공정을 거친 station_name\n\n- time_diff : 해당 product가 처음 시작한 timestamp와 마지막으로 끝낸 timestamp의 차이 (공정에 걸린 시간)","metadata":{"papermill":{"duration":0.013331,"end_time":"2022-11-10T14:43:39.626384","exception":false,"start_time":"2022-11-10T14:43:39.613053","status":"completed"},"tags":[],"id":"54c2d857"}},{"cell_type":"code","source":"# file I/O\nfor chunk in pd.read_csv(base_path + 'train_date.csv.zip', chunksize=300000):\n    ","metadata":{"papermill":{"duration":45.638743,"end_time":"2022-11-10T14:44:25.580077","exception":false,"start_time":"2022-11-10T14:43:39.941334","status":"completed"},"tags":[],"id":"1a94d7b1","outputId":"e45bd9e7-a180-4793-9ddb-0490d7d13132","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 전체 데이터에 결측치가 많을 땐 어떻게 해야할까요?\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. EDA","metadata":{"id":"e41fbe06","papermill":{"duration":0.015321,"end_time":"2022-11-10T14:45:07.174045","exception":false,"start_time":"2022-11-10T14:45:07.158724","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"- 데이터에서 찾아야 하는 기초적인 내용들을 확인합니다.\n\n\n- class imbalance, target distribution, outlier, correlation을 확인합니다.","metadata":{"id":"0ce7894b","papermill":{"duration":0.015944,"end_time":"2022-11-10T14:45:07.205424","exception":false,"start_time":"2022-11-10T14:45:07.189480","status":"completed"},"tags":[]}},{"cell_type":"code","source":"## On your Own\ndisplay(X_train.Response.value_counts()) # 0.995 : 0.005 = 995 : 5 = 199 : 1\nsns.countplot(data=X_train, x=\"Response\")","metadata":{"id":"347eed9a","papermill":{"duration":0.337856,"end_time":"2022-11-10T14:45:07.559729","exception":false,"start_time":"2022-11-10T14:45:07.221873","status":"completed"},"tags":[],"outputId":"5a66c341-04d4-4241-c18c-f8270f608606","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"이런 식으로 여러가지 그래프를 그려가며, 데이터에 대한 인사이트를 얻습니다!","metadata":{"id":"d2e6e245","papermill":{"duration":0.015784,"end_time":"2022-11-10T14:45:07.592025","exception":false,"start_time":"2022-11-10T14:45:07.576241","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 3. 전처리\n\nClass Imbalance가 존재하는 경우에는 data sampling 기법을 이용하여 문제를 해결합니다.\n- majority class : 다수가 속해있는 class\n\n- minority class : 소수가 속해있는 class\n\n\n\n**1) Under sampling : majority class의 수를 minority class로 맞춰주기 위해서 random sampling을 수행하는 방법.**\n\ne.g. 1176868 : 6879 -> 6879 : 6879\n\n\n**2) Over sampling : minority class에서 데이터를 생성하여 majority class의 수를 맞춰주는 방법.**\n\ne.g. 1176868 : 6879 -> 1176868 : 1176868\n\n\n**3) Hybrid approach : under sampling & over sampling을 써서 적당한 수로 맞춰주는 방법.**\n\ne.g. 1176868 : 6879 -> 20000 : 20000","metadata":{"id":"b90cb94b","papermill":{"duration":0.015639,"end_time":"2022-11-10T14:45:07.623707","exception":false,"start_time":"2022-11-10T14:45:07.608068","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 모든 데이터가 NaN이라서, time_diff가 NaN이고 train_date에 아무 정보가 없는 데이터들을 제거합니다.\nX_train = X_train.dropna() # 1183767 - 582\nX_train","metadata":{"papermill":{"duration":0.578611,"end_time":"2022-11-10T14:45:08.218374","exception":false,"start_time":"2022-11-10T14:45:07.639763","status":"completed"},"tags":[],"id":"431e73bf","outputId":"c7fd50b2-53c4-477f-c685-8974092af730","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem(df):\n    for col, dtype in zip(df.dtypes.index, df.dtypes.values):\n        if dtype == 'int64':\n            df[col] = df[col].astype(np.int32)\n        elif dtype == 'float64':\n            df[col] = df[col].astype(np.float32)\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-11-16T06:31:23.149490Z","iopub.execute_input":"2023-11-16T06:31:23.150631Z","iopub.status.idle":"2023-11-16T06:31:23.156994Z","shell.execute_reply.started":"2023-11-16T06:31:23.150573Z","shell.execute_reply":"2023-11-16T06:31:23.155968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_numeric = pd.read_csv('/kaggle/input/bosch-unzipped/train_numeric.csv')","metadata":{"execution":{"iopub.status.busy":"2023-11-16T06:31:23.634209Z","iopub.execute_input":"2023-11-16T06:31:23.635213Z","iopub.status.idle":"2023-11-16T06:33:32.040534Z","shell.execute_reply.started":"2023-11-16T06:31:23.635173Z","shell.execute_reply":"2023-11-16T06:33:32.038887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_numeric = reduce_mem(train_numeric)\ntrain_numeric.info()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T06:33:32.042998Z","iopub.execute_input":"2023-11-16T06:33:32.043311Z","iopub.status.idle":"2023-11-16T06:33:34.374413Z","shell.execute_reply.started":"2023-11-16T06:33:32.043285Z","shell.execute_reply":"2023-11-16T06:33:34.373368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# n_normal = 1176868\n# n_abnormal = 6879\nsampling = 'hybrid'","metadata":{"execution":{"iopub.status.busy":"2023-11-16T06:33:34.375742Z","iopub.execute_input":"2023-11-16T06:33:34.376054Z","iopub.status.idle":"2023-11-16T06:33:34.380432Z","shell.execute_reply.started":"2023-11-16T06:33:34.376027Z","shell.execute_reply":"2023-11-16T06:33:34.379388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 1) Undersampling\nif sampling == 'under':\n    N = 20000\n    normal = train_numeric[train_numeric.Response == 0]\n    abnormal = train_numeric[train_numeric.Response == 1]\n    sample_normal = normal.sample(n=N)\n    data = pd.concat([sample_normal, abnormal]).reset_index(drop=True)\n    data = data.fillna(0)\n    \n    #del train_numeric\n    del normal\n    del abnormal\n    del sample_normal\n    gc.collect()  # main memory에 남아있는 데이터를 모두 지워서, 여유 메모리를 확보함.","metadata":{"papermill":{"duration":0.027254,"end_time":"2022-11-10T14:45:08.263068","exception":false,"start_time":"2022-11-10T14:45:08.235814","status":"completed"},"tags":[],"id":"ad8c39b0","execution":{"iopub.status.busy":"2023-11-16T06:33:34.382750Z","iopub.execute_input":"2023-11-16T06:33:34.383144Z","iopub.status.idle":"2023-11-16T06:33:34.393611Z","shell.execute_reply.started":"2023-11-16T06:33:34.383115Z","shell.execute_reply":"2023-11-16T06:33:34.392438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 결측치의 개수 자체는 합/불 판정에 큰 의미가 없다. ---> 모든 결측치를 0으로 처리.\n# data.isnull().sum(axis=1)[:6879].mean(), data.isnull().sum(axis=1)[6879:].mean()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T06:33:34.394892Z","iopub.execute_input":"2023-11-16T06:33:34.395306Z","iopub.status.idle":"2023-11-16T06:33:34.403694Z","shell.execute_reply.started":"2023-11-16T06:33:34.395263Z","shell.execute_reply":"2023-11-16T06:33:34.402887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if sampling == 'over':\n    # 2) Oversampling - SMOTE (only numeric features)\n    temp = train_numeric.iloc[:, [0, 1, 2, 3, 4, 5, -1]].fillna(0)\n    X = temp.drop('Response', axis=1) # features\n    y = temp.Response # target\n    smote = SMOTE()\n    # y를 보고 자동으로 majority, minority class를 찾아서 minority class 데이터를 학습해서, 클래스의 비율을 1:1로 만들어줌.\n    data = smote.fit_resample(X, y)\n    \n    del temp\n    del X\n    del y\n    gc.collect()","metadata":{"papermill":{"duration":0.026627,"end_time":"2022-11-10T14:45:08.306271","exception":false,"start_time":"2022-11-10T14:45:08.279644","status":"completed"},"tags":[],"id":"811225e4","execution":{"iopub.status.busy":"2023-11-16T06:33:34.404869Z","iopub.execute_input":"2023-11-16T06:33:34.405365Z","iopub.status.idle":"2023-11-16T06:33:34.418381Z","shell.execute_reply.started":"2023-11-16T06:33:34.405335Z","shell.execute_reply":"2023-11-16T06:33:34.417245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if sampling == 'hybrid':\n    # 3) Hybrid approach\n    N = 20000\n    normal = train_numeric[train_numeric.Response == 0]\n    abnormal = train_numeric[train_numeric.Response == 1]\n    sample_normal = normal.sample(n=N)\n    data = pd.concat([sample_normal, abnormal]).reset_index(drop=True)\n    data = data.fillna(0) # N + 6879\n    \n    X = data.drop('Response', axis=1) # features\n    y = data.Response # target\n    smote = SMOTE()\n    # y를 보고 자동으로 majority, minority class를 찾아서 minority class 데이터를 학습해서, 클래스의 비율을 1:1로 만들어줌.\n    data = smote.fit_resample(X, y)\n    \n    del X\n    del y\n    del normal\n    del abnormal\n    del sample_normal\n    gc.collect()  # main memory에 남아있는 데이터를 모두 지워서, 여유 메모리를 확보함.","metadata":{"papermill":{"duration":1.661906,"end_time":"2022-11-10T14:45:09.984509","exception":false,"start_time":"2022-11-10T14:45:08.322603","status":"completed"},"tags":[],"id":"03c4af34","outputId":"16a902a3-4d74-48b6-e9af-74a99dcbf530","execution":{"iopub.status.busy":"2023-11-16T06:33:34.419773Z","iopub.execute_input":"2023-11-16T06:33:34.420136Z","iopub.status.idle":"2023-11-16T06:33:42.346657Z","shell.execute_reply.started":"2023-11-16T06:33:34.420104Z","shell.execute_reply":"2023-11-16T06:33:42.345756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_numeric\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T06:33:42.347807Z","iopub.execute_input":"2023-11-16T06:33:42.348108Z","iopub.status.idle":"2023-11-16T06:33:42.601134Z","shell.execute_reply.started":"2023-11-16T06:33:42.348081Z","shell.execute_reply":"2023-11-16T06:33:42.599995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4. 학습 데이터 분할","metadata":{"id":"3f502828","papermill":{"duration":0.0169,"end_time":"2022-11-10T14:45:10.069019","exception":false,"start_time":"2022-11-10T14:45:10.052119","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# custom dataset & dataloader 정의\nclass MyDataset(Dataset):   # custom dataset을 만들어야하는 경우, Dataset class를 상속받는 새로운 class를 정의해야합니다.\n    def __init__(self, df, mode = \"train\"):\n        self.df = df\n        self.mode = mode\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, index):\n        if self.mode == \"train\":\n            X = torch.tensor(self.df.iloc[index, :-1].values, dtype=torch.float)\n            y = torch.tensor(self.df.iloc[index, -1], dtype=torch.long)\n            return X, y\n        else:\n            X = torch.tensor(self.df.iloc[index, :].values, dtype=torch.float)\n            return X","metadata":{"execution":{"iopub.status.busy":"2023-11-16T06:35:42.993906Z","iopub.execute_input":"2023-11-16T06:35:42.995173Z","iopub.status.idle":"2023-11-16T06:35:43.002949Z","shell.execute_reply.started":"2023-11-16T06:35:42.995129Z","shell.execute_reply":"2023-11-16T06:35:43.001875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_loader, valid_loader를 정의하는 함수.\ndef prepare_loader(train, ratio, batch_size):\n\n    X = train.drop(['Id','Response'], axis=1)\n    y = train['Response']\n    # 4개의 클래스 비율이 동일하게, train & validation 데이터가 나눠짐.\n    X_train, X_val, y_train, y_val = train_test_split(X, y, train_size=ratio, random_state=42, stratify=y)\n    train_df = pd.concat([X_train, y_train], axis=1).reset_index(drop=True)\n    valid_df = pd.concat([X_val, y_val], axis=1).reset_index(drop=True)\n    \n    ## train, valid -> Dataset\n    train_ds = MyDataset(train_df, mode = \"train\")\n\n    valid_ds = MyDataset(valid_df, mode = \"train\")\n\n    # Dataset -> DataLoader\n    train_loader = DataLoader(train_ds,\n                              batch_size = batch_size,\n                              shuffle = True,\n                              pin_memory = True)\n\n    valid_loader = DataLoader(valid_ds,\n                              batch_size = batch_size,\n                              shuffle = False,\n                              pin_memory = True)\n\n    print(\"DataLoader Completed\")\n    return train_loader, valid_loader","metadata":{"execution":{"iopub.status.busy":"2023-11-16T06:35:46.319424Z","iopub.execute_input":"2023-11-16T06:35:46.320183Z","iopub.status.idle":"2023-11-16T06:35:46.328921Z","shell.execute_reply.started":"2023-11-16T06:35:46.320147Z","shell.execute_reply":"2023-11-16T06:35:46.327872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.concat([data[0], data[1]], axis=1)\ndata","metadata":{"execution":{"iopub.status.busy":"2023-11-16T06:33:42.629360Z","iopub.execute_input":"2023-11-16T06:33:42.629732Z","iopub.status.idle":"2023-11-16T06:33:42.805700Z","shell.execute_reply.started":"2023-11-16T06:33:42.629704Z","shell.execute_reply":"2023-11-16T06:33:42.804611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\nscaler = MinMaxScaler()\ntemp = scaler.fit_transform(data)\ndata = pd.DataFrame(data=temp, columns=data.columns)\ndata","metadata":{"execution":{"iopub.status.busy":"2023-11-16T07:01:09.852666Z","iopub.execute_input":"2023-11-16T07:01:09.853584Z","iopub.status.idle":"2023-11-16T07:01:10.330041Z","shell.execute_reply.started":"2023-11-16T07:01:09.853534Z","shell.execute_reply":"2023-11-16T07:01:10.329030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainloader, validloader = prepare_loader(train = data, ratio = .75, batch_size = 256)","metadata":{"execution":{"iopub.status.busy":"2023-11-16T07:02:17.393116Z","iopub.execute_input":"2023-11-16T07:02:17.394138Z","iopub.status.idle":"2023-11-16T07:02:17.989755Z","shell.execute_reply.started":"2023-11-16T07:02:17.394102Z","shell.execute_reply":"2023-11-16T07:02:17.988514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 5. 학습 및 평가","metadata":{"id":"eb8950a4","papermill":{"duration":0.015979,"end_time":"2022-11-10T14:45:10.170437","exception":false,"start_time":"2022-11-10T14:45:10.154458","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 모델 정의 (Deep AutoEncoder)\n# X -> enc1 -> latent -> dec1 -> X'\n\n# Classifier\n# X -> enc1 -> fc -> 1\nclass Autoencoder(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.enc1 = nn.Linear(968, 256)\n        self.enc2 = nn.Linear(256, 20) # z\n        \n        self.dec1 = nn.Linear(20, 256)\n        self.dec2 = nn.Linear(256, 968)\n        \n        self.relu = nn.ReLU()\n        self.sigmoid = nn.Sigmoid() # tensor의 마지막 차원에 적용.\n        \n    def encoder(self, x):\n        x = self.relu(self.enc1(x))\n        x = self.relu(self.enc2(x))\n        return x\n    \n    def decoder(self, x):\n        x = self.relu(self.dec1(x))\n        x = self.sigmoid(self.dec2(x)) # output의 출력 범위를 [0, 1]로 제한.\n        return x\n        \n    def forward(self, x):\n        z = self.encoder(x)\n        X_hat = self.decoder(z)\n        return X_hat, z","metadata":{"execution":{"iopub.status.busy":"2023-11-16T07:33:19.492150Z","iopub.execute_input":"2023-11-16T07:33:19.492963Z","iopub.status.idle":"2023-11-16T07:33:19.502430Z","shell.execute_reply.started":"2023-11-16T07:33:19.492925Z","shell.execute_reply":"2023-11-16T07:33:19.501271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchinfo import summary\nmodel = Autoencoder().to('cuda')\nsummary(model)","metadata":{"execution":{"iopub.status.busy":"2023-11-16T07:33:21.418265Z","iopub.execute_input":"2023-11-16T07:33:21.418944Z","iopub.status.idle":"2023-11-16T07:33:21.432032Z","shell.execute_reply.started":"2023-11-16T07:33:21.418908Z","shell.execute_reply":"2023-11-16T07:33:21.431126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# optimizer, criterion, lr_scheduler 정의\noptimizer = torch.optim.Adam(model.parameters(),\n                              lr=1e-3)\ncriterion = nn.MSELoss()\nCEloss = nn.BCELoss() # binary classification\n#scheduler = ","metadata":{"execution":{"iopub.status.busy":"2023-11-16T07:34:06.880142Z","iopub.execute_input":"2023-11-16T07:34:06.881030Z","iopub.status.idle":"2023-11-16T07:34:06.887067Z","shell.execute_reply.started":"2023-11-16T07:34:06.880992Z","shell.execute_reply":"2023-11-16T07:34:06.885547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ff check\ntx, ty = next(iter(trainloader))\ntx = tx.to('cuda')\nty = ty.to('cuda')\nX_hat, z = model(tx)\nloss = criterion(X_hat, tx)\noutputs.shape, loss","metadata":{"execution":{"iopub.status.busy":"2023-11-16T07:36:54.458352Z","iopub.execute_input":"2023-11-16T07:36:54.459400Z","iopub.status.idle":"2023-11-16T07:36:54.520486Z","shell.execute_reply.started":"2023-11-16T07:36:54.459364Z","shell.execute_reply":"2023-11-16T07:36:54.519511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_val_loss = 123456789.0   # worst value\ntol = 1e-8\npatience = 25\npatience_count = 0\n# Early stopping의 기준 : best_val_loss보다 tol 이하의 차이를 가지는 patience_count가 patience보다 같아지면 stop.\ntrain_losses = []\nval_losses = []\nepoch_end = 0\ndevice = 'cuda'\n\nfor epoch in tqdm(range(50)):\n    train_loss = 0.0\n    val_loss = 0.0\n\n    # training\n    model.train()  # training mode\n    for idx, data in enumerate(trainloader):\n        # dtype을 float32 -> float16으로 변환해서 메모리 사용량을 절반으로 줄이는 코드.\n        with torch.autocast(device_type='cuda', dtype=torch.float16):  ## tesla V100\n\n            X, y = data[0].to(device), data[1].to(device)\n            X_hat, z = model(X)\n            loss = criterion(X_hat, X)\n\n        optimizer.zero_grad()  # backprop할 때, gradient를 초기화해줌.\n        loss.backward()\n        optimizer.step()\n\n        train_loss = train_loss + loss.item()\n\n    #scheduler.step()\n    train_loss = train_loss / len(trainloader)\n    train_losses.append(train_loss)\n    \n    if (epoch % 1000) == 0:\n        print(f\"Epoch {epoch} ---> Train loss : {train_loss:.4f}\")\n\n    # validation\n    model.eval()  # predict mode (dropout, batchnorm 같은 train/predict에서 다르게 동작하는 모듈을 전환해주는 함수.)\n    with torch.no_grad(): # locally disabling gradient computation\n        for idx, data in enumerate(validloader):\n            with torch.autocast(device_type='cuda', dtype=torch.float16):\n                X, y = data[0].to(device), data[1].to(device)\n                X_hat, z = model(X)\n                loss = criterion(X_hat, X)\n                val_loss = val_loss + loss.item()\n\n        val_loss = val_loss / len(validloader)\n        val_losses.append(val_loss)\n        \n        if (epoch % 1000) == 0:\n            print(f\"Epoch {epoch} ---> Validation loss : {val_loss:.6f}\")\n\n    # check best (val loss의 변화량이 tol보다 클 때)\n    if best_val_loss - val_loss > tol:\n        best_val_loss = val_loss\n        print(f\"Epoch {epoch} ---> (Best)Validation loss : {best_val_loss:.6f}\")\n        patience_count = 0\n    else:\n        patience_count = patience_count + 1\n\n    if patience_count > patience:\n        print(f\"Early stopping at {epoch} epoch.\")\n        epoch_end = epoch\n        break","metadata":{"execution":{"iopub.status.busy":"2023-11-16T07:40:41.625537Z","iopub.execute_input":"2023-11-16T07:40:41.626478Z","iopub.status.idle":"2023-11-16T07:46:48.821995Z","shell.execute_reply.started":"2023-11-16T07:40:41.626442Z","shell.execute_reply":"2023-11-16T07:46:48.820940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## TO-DO ##\n# (pseudo-code)\n###### classifier 만들기!! #####\n# model.encoder(x)\n\n# class Classifier(nn.Module):\n#     ~~~~\n#     ~~~~\n# classifier = Classifier().to(device)\n# for data in trainloader:\n    # ~~~~\n    # _, z = model(x)\n    # y_hat = classifier(z)\n    # loss = CEloss(y_hat, y)\n    # F1score = torchmetrics.F1Score(~~~~)\n\n###### SVDD ######\n# MSELoss를 수정해서, batch단위로 출력되게 만듬.\n## SVDD식으로 학습을 하기 위해서는 AE를 학습할 때, 정상 데이터만 학습해야함. (threshold를 구하기 위해서)\n# with torch.no_grad():\n    # for data in trainloader: (주의: 이 때 testloader에는 정상 & 불량이 모두 포함되어 있어도 됨.)\n        # ~~~~\n        # X_hat, _ = model(X)\n        # loss = criterion(X_hat, X)   (batch_size, )\n        # t = loss.max()\n        # if threshold < t:\n        #    threshold = t   (학습시 구한 최대 loss)\n\n# with torch.no_grad():\n    # for data in testloader:\n        # X_hat, _ = model(X)\n        # loss = criterion(X_hat, X)   (batch_size, )\n        # if threshold > loss:\n        #    y_hat = 1    (불량)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 7. 테스트 및 제출 파일 생성","metadata":{"id":"70b1e9a7","papermill":{"duration":0.020451,"end_time":"2022-11-10T16:07:45.148892","exception":false,"start_time":"2022-11-10T16:07:45.128441","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_num_part = pd.read_csv(base_path + 'test_numeric.csv.zip',\n                           nrows=1000)\ntest_date_part = pd.read_csv(base_path + 'test_date.csv.zip',\n                            nrows=1000)","metadata":{"papermill":{"duration":0.479587,"end_time":"2022-11-10T16:07:45.847970","exception":false,"start_time":"2022-11-10T16:07:45.368383","status":"completed"},"tags":[],"id":"084526a4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## TO-DO\n## testloader를 사용해서, inference 코드 만들고 submission.csv 파일 완성하기!","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(base_path + \"sample_submission.csv.zip\")\nsubmission['Response'] = preds","metadata":{"id":"dbceb995","papermill":{"duration":0.285324,"end_time":"2022-11-10T16:10:35.910473","exception":false,"start_time":"2022-11-10T16:10:35.625149","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","metadata":{"id":"8be4ba3f","papermill":{"duration":1.24429,"end_time":"2022-11-10T16:10:37.178722","exception":false,"start_time":"2022-11-10T16:10:35.934432","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Isolation Forest\n\n- anomaly detection의 대표적인 unsupervised learning 기법!","metadata":{"id":"uV0v6IEHFqp3"}},{"cell_type":"code","source":"from sklearn.ensemble import IsolationForest\n\niforest = IsolationForest(n_estimators=100, # 학습하는 트리 개수\n                          max_samples='auto', # min(256, n_samples), 트리별 학습 sample 수\n                          contamination='auto', # float 값을 넣으면 해당하는 %의 데이터를 outlier로 줌. e.g. 0.01 = 1%\n                          n_jobs=-1, # 모든 core를 사용\n                          random_state=42)\n#iforest.fit(X_train)\npreds = iforest.fit_predict(X_train) # fit & predict\npreds","metadata":{"id":"R1jVMRMJFqQR","outputId":"b0653cd9-9195-4d7c-aa29-9f0b957586cb"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" #iforest의 결과 -> 1 : normal, -1 : abnormal\n # 원본 데이터는 -> 0:normal, 1 : abnormal\npreds = pd.Series(preds).map({1 : 0, -1 : 1})\npreds","metadata":{"id":"E-BsZ4cHItFm","outputId":"81ca0714-006e-493a-b59c-c2ccfe2ce003"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds.value_counts()","metadata":{"id":"htVZ8H7QJQEL","outputId":"eb5d506b-687f-40d4-98ea-12ebb1dff899"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Train MCC : %.4f\" % evaluation_metric(y_train, preds))","metadata":{"id":"1T3NC_aiIcEK","outputId":"c6385371-ddd5-412e-c751-5373dd215c28"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"l-x1lO-dIqKV"},"execution_count":null,"outputs":[]}]}