{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"papermill":{"default_parameters":{},"duration":5232.034539,"end_time":"2022-11-10T16:10:38.379903","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2022-11-10T14:43:26.345364","version":"2.3.4"},"colab":{"provenance":[]},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":5357,"databundleVersionId":38056,"sourceType":"competition"},{"sourceId":6978076,"sourceType":"datasetVersion","datasetId":4009850}],"dockerImageVersionId":30588,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 불량 검출 문제를 풀기 위한 접근 방법\n\n1. 지도학습 문제로 정의 : binary classification (정상 / 불량)\n\n> 두 개의 클래스 사이의 큰 불균형이 존재하기 때문에 해결해야함.\n\n\n2. 비지도학습 문제로 정의 : clustering (2가지로 자동 분류)\n\n> K-means보단, DBSCAN 같은 모델이 주로 사용됨.\n\n\n3. Anomaly Detection 문제로 정의 : 불량을 outlier로 정의해서 대부분의 데이터(majority)와 나머지를 찾는 문제로 정의.\n\n> 다양한 anomaly detection 모델을 사용함.","metadata":{"id":"e34fe8bf","papermill":{"duration":0.015043,"end_time":"2022-11-10T14:43:35.604553","exception":false,"start_time":"2022-11-10T14:43:35.58951","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## 1. 라이브러리, 데이터 불러오기","metadata":{"id":"222f8ad3","papermill":{"duration":0.014616,"end_time":"2022-11-10T14:43:35.66071","exception":false,"start_time":"2022-11-10T14:43:35.646094","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install optuna torchinfo torchmetrics","metadata":{"id":"srSiW1GUjrxY","outputId":"4b4efcd7-475a-4ca4-98b5-176e48b05cf1","execution":{"iopub.status.busy":"2023-11-16T02:14:21.774787Z","iopub.execute_input":"2023-11-16T02:14:21.775107Z","iopub.status.idle":"2023-11-16T02:14:35.758808Z","shell.execute_reply.started":"2023-11-16T02:14:21.77508Z","shell.execute_reply":"2023-11-16T02:14:35.757631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 데이터분석 4종 세트\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# 모델들, 성능 평가\n# (저는 일반적으로 정형데이터로 머신러닝 분석할 때는 이 2개 모델은 그냥 돌려봅니다. 특히 RF가 테스트하기 좋습니다.)\nfrom lightgbm.sklearn import LGBMClassifier\n\n# KFold(CV), partial : optuna를 사용하기 위함\nfrom sklearn.model_selection import KFold\nfrom functools import partial\n\n# hyper-parameter tuning을 위한 라이브러리, optuna\nimport optuna","metadata":{"id":"5ba77e23","papermill":{"duration":3.342311,"end_time":"2022-11-10T14:43:39.017482","exception":false,"start_time":"2022-11-10T14:43:35.675171","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-16T02:34:13.508944Z","iopub.execute_input":"2023-11-16T02:34:13.509388Z","iopub.status.idle":"2023-11-16T02:34:18.137827Z","shell.execute_reply.started":"2023-11-16T02:34:13.509352Z","shell.execute_reply":"2023-11-16T02:34:18.136768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nimport os\nimport gc\nimport time\nimport random\nimport string\n\nimport copy\nfrom copy import deepcopy\n\nimport torchmetrics\nfrom torchmetrics.classification import BinaryF1Score\n\nfrom sklearn.model_selection import train_test_split\n\n## Pytorch Import\nimport torch\nimport torch.nn as nn\n\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\n\n# Utils\nfrom tqdm.auto import tqdm, trange\n\n# Suppress warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2023-11-16T02:34:18.139531Z","iopub.execute_input":"2023-11-16T02:34:18.139869Z","iopub.status.idle":"2023-11-16T02:34:23.990414Z","shell.execute_reply.started":"2023-11-16T02:34:18.139824Z","shell.execute_reply":"2023-11-16T02:34:23.989362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# flag setting\nsampling = 'hybrid' ## 'under' or 'over' or 'hybrid'","metadata":{"execution":{"iopub.execute_input":"2022-11-10T14:43:39.049179Z","iopub.status.busy":"2022-11-10T14:43:39.047912Z","iopub.status.idle":"2022-11-10T14:43:39.05342Z","shell.execute_reply":"2022-11-10T14:43:39.052594Z"},"id":"3baf9b7e","papermill":{"duration":0.022911,"end_time":"2022-11-10T14:43:39.05573","exception":false,"start_time":"2022-11-10T14:43:39.032819","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 데이터를 불러옵니다.\nbase_path = \"/kaggle/input/bosch-production-line-performance/\"\ntrain_cat_part = pd.read_csv(base_path + 'train_categorical.csv.zip',\n                            nrows=1000)\ntrain_num_part = pd.read_csv(base_path + 'train_numeric.csv.zip',\n                            nrows=1000)\ntrain_date_part = pd.read_csv(base_path + 'train_date.csv.zip',\n                             nrows=1000)\n\nprint(train_cat_part.shape, train_num_part.shape, train_date_part.shape)","metadata":{"id":"ff89465f","papermill":{"duration":0.427331,"end_time":"2022-11-10T14:43:39.49708","exception":false,"start_time":"2022-11-10T14:43:39.069749","status":"completed"},"tags":[],"outputId":"961a25ae-296b-4b98-ca34-d10af5ca4a46","execution":{"iopub.status.busy":"2023-11-16T02:35:22.187244Z","iopub.execute_input":"2023-11-16T02:35:22.188015Z","iopub.status.idle":"2023-11-16T02:35:22.989151Z","shell.execute_reply.started":"2023-11-16T02:35:22.187981Z","shell.execute_reply":"2023-11-16T02:35:22.988216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp = pd.read_csv(base_path + 'train_numeric.csv.zip')\ntemp","metadata":{"execution":{"iopub.status.busy":"2023-11-16T02:37:18.687707Z","iopub.execute_input":"2023-11-16T02:37:18.68813Z","iopub.status.idle":"2023-11-16T02:39:24.367096Z","shell.execute_reply.started":"2023-11-16T02:37:18.688099Z","shell.execute_reply":"2023-11-16T02:39:24.365984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count = 0\n\nfor row in temp.isnull().mean():\n    if row >= 0.9:\n        print(row)\n        count += 1\n        \nprint(count)","metadata":{"execution":{"iopub.status.busy":"2023-11-16T02:45:37.831478Z","iopub.execute_input":"2023-11-16T02:45:37.831916Z","iopub.status.idle":"2023-11-16T02:45:39.608602Z","shell.execute_reply.started":"2023-11-16T02:45:37.831882Z","shell.execute_reply":"2023-11-16T02:45:39.607575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"categorical data는 EDA 결과 분석에 큰 도움이 없어, 사용하지 않겠습니다.","metadata":{"papermill":{"duration":0.013428,"end_time":"2022-11-10T14:43:39.525135","exception":false,"start_time":"2022-11-10T14:43:39.511707","status":"completed"},"tags":[],"id":"6183159f"}},{"cell_type":"code","source":"# total = train_cat_part.count() * 0\n# N = 0\n\n# for chunk in pd.read_csv('../input/bosch-production-line-performance/train_categorical.csv.zip',\n#                         chunksize=100000):  # chunksize가 클수록, 속도는 빨라지고 메모리 사용량이 늘어납니다.\n\n#     total = total + chunk.isnull().sum()  # 전체 데이터의 column별 결측치 개수\n#     N = N + len(chunk)  # 전체 데이터의 row 수\n\n# display(total)\n# print(N)\n# usecols = total.index[(total / N) < 0.5] # train_categorical.csv에 있는 결측치가 50% 미만인 column들.\n# usecols","metadata":{"execution":{"iopub.execute_input":"2022-11-10T14:43:39.555145Z","iopub.status.busy":"2022-11-10T14:43:39.554662Z","iopub.status.idle":"2022-11-10T14:43:39.560009Z","shell.execute_reply":"2022-11-10T14:43:39.558766Z"},"papermill":{"duration":0.023711,"end_time":"2022-11-10T14:43:39.562587","exception":false,"start_time":"2022-11-10T14:43:39.538876","status":"completed"},"tags":[],"id":"14b8b9ed"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_cat = pd.read_csv('../input/bosch-production-line-performance/train_categorical.csv.zip',\n#                        usecols=usecols)\n# train_cat","metadata":{"execution":{"iopub.execute_input":"2022-11-10T14:43:39.592418Z","iopub.status.busy":"2022-11-10T14:43:39.59198Z","iopub.status.idle":"2022-11-10T14:43:39.596939Z","shell.execute_reply":"2022-11-10T14:43:39.595608Z"},"papermill":{"duration":0.022751,"end_time":"2022-11-10T14:43:39.599204","exception":false,"start_time":"2022-11-10T14:43:39.576453","status":"completed"},"tags":[],"id":"9773eef8"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"train_date.csv에서는 파생 변수를 만들어서 사용합니다.\n\n> 1. 각 station별로 측정 시간이 모두 있거나, 없거나의 케이스만 존재한다.\n>> 데이터 중에서 station별로 하나의 column만 뽑아서 체크해도 괜찮다.\n\n> 2. 각 station별로 시간이 모두 같지는 않다. 중간에 측정시간이 바뀌는 케이스도 있다.\n>> 데이터 별로 min, max값을 계산하면 된다.\n\n- start_station : 해당 product가 처음으로 공정을 거친 station_name\n\n- end_station : 해당 product가 마지막으로 공정을 거친 station_name\n\n- time_diff : 해당 product가 처음 시작한 timestamp와 마지막으로 끝낸 timestamp의 차이 (공정에 걸린 시간)","metadata":{"papermill":{"duration":0.013331,"end_time":"2022-11-10T14:43:39.626384","exception":false,"start_time":"2022-11-10T14:43:39.613053","status":"completed"},"tags":[],"id":"54c2d857"}},{"cell_type":"code","source":"# file I/O\nfor chunk in pd.read_csv(base_path + 'train_date.csv.zip', chunksize=300000):\n    ","metadata":{"execution":{"iopub.execute_input":"2022-11-10T14:43:39.958447Z","iopub.status.busy":"2022-11-10T14:43:39.957941Z","iopub.status.idle":"2022-11-10T14:44:25.562769Z","shell.execute_reply":"2022-11-10T14:44:25.561412Z"},"papermill":{"duration":45.638743,"end_time":"2022-11-10T14:44:25.580077","exception":false,"start_time":"2022-11-10T14:43:39.941334","status":"completed"},"tags":[],"id":"1a94d7b1","outputId":"e45bd9e7-a180-4793-9ddb-0490d7d13132"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 전체 데이터에 결측치가 많을 땐 어떻게 해야할까요?\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. EDA","metadata":{"id":"e41fbe06","papermill":{"duration":0.015321,"end_time":"2022-11-10T14:45:07.174045","exception":false,"start_time":"2022-11-10T14:45:07.158724","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"- 데이터에서 찾아야 하는 기초적인 내용들을 확인합니다.\n\n\n- class imbalance, target distribution, outlier, correlation을 확인합니다.","metadata":{"id":"0ce7894b","papermill":{"duration":0.015944,"end_time":"2022-11-10T14:45:07.205424","exception":false,"start_time":"2022-11-10T14:45:07.18948","status":"completed"},"tags":[]}},{"cell_type":"code","source":"## On your Own\ndisplay(X_train.Response.value_counts()) # 0.995 : 0.005 = 995 : 5 = 199 : 1\nsns.countplot(data=X_train, x=\"Response\")","metadata":{"execution":{"iopub.execute_input":"2022-11-10T14:45:07.239499Z","iopub.status.busy":"2022-11-10T14:45:07.238723Z","iopub.status.idle":"2022-11-10T14:45:07.55755Z","shell.execute_reply":"2022-11-10T14:45:07.556695Z"},"id":"347eed9a","papermill":{"duration":0.337856,"end_time":"2022-11-10T14:45:07.559729","exception":false,"start_time":"2022-11-10T14:45:07.221873","status":"completed"},"tags":[],"outputId":"5a66c341-04d4-4241-c18c-f8270f608606"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"이런 식으로 여러가지 그래프를 그려가며, 데이터에 대한 인사이트를 얻습니다!","metadata":{"id":"d2e6e245","papermill":{"duration":0.015784,"end_time":"2022-11-10T14:45:07.592025","exception":false,"start_time":"2022-11-10T14:45:07.576241","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### 3. 전처리\n\nClass Imbalance가 존재하는 경우에는 data sampling 기법을 이용하여 문제를 해결합니다.\n- majority class : 다수가 속해있는 class\n\n- minority class : 소수가 속해있는 class\n\n\n\n**1) Under sampling : majority class의 수를 minority class로 맞춰주기 위해서 random sampling을 수행하는 방법.**\n\ne.g. 1176868 : 6879 -> 6879 : 6879\n\n\n**2) Over sampling : minority class에서 데이터를 생성하여 majority class의 수를 맞춰주는 방법.**\n\ne.g. 1176868 : 6879 -> 1176868 : 1176868\n\n\n**3) Hybrid approach : under sampling & over sampling을 써서 적당한 수로 맞춰주는 방법.**\n\ne.g. 1176868 : 6879 -> 20000 : 20000","metadata":{"id":"b90cb94b","papermill":{"duration":0.015639,"end_time":"2022-11-10T14:45:07.623707","exception":false,"start_time":"2022-11-10T14:45:07.608068","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 모든 데이터가 NaN이라서, time_diff가 NaN이고 train_date에 아무 정보가 없는 데이터들을 제거합니다.\nX_train = X_train.dropna() # 1183767 - 582\nX_train","metadata":{"execution":{"iopub.execute_input":"2022-11-10T14:45:07.657917Z","iopub.status.busy":"2022-11-10T14:45:07.657441Z","iopub.status.idle":"2022-11-10T14:45:08.215604Z","shell.execute_reply":"2022-11-10T14:45:08.21435Z"},"papermill":{"duration":0.578611,"end_time":"2022-11-10T14:45:08.218374","exception":false,"start_time":"2022-11-10T14:45:07.639763","status":"completed"},"tags":[],"id":"431e73bf","outputId":"c7fd50b2-53c4-477f-c685-8974092af730"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 1) Undersampling\nif sampling == 'under':\n","metadata":{"execution":{"iopub.execute_input":"2022-11-10T14:45:08.254692Z","iopub.status.busy":"2022-11-10T14:45:08.254269Z","iopub.status.idle":"2022-11-10T14:45:08.260799Z","shell.execute_reply":"2022-11-10T14:45:08.259555Z"},"papermill":{"duration":0.027254,"end_time":"2022-11-10T14:45:08.263068","exception":false,"start_time":"2022-11-10T14:45:08.235814","status":"completed"},"tags":[],"id":"ad8c39b0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if sampling == 'over':\n    # 2) Oversampling - SMOTE (only numeric features)\n    from imblearn.over_sampling import SMOTE","metadata":{"execution":{"iopub.execute_input":"2022-11-10T14:45:08.29828Z","iopub.status.busy":"2022-11-10T14:45:08.297303Z","iopub.status.idle":"2022-11-10T14:45:08.303972Z","shell.execute_reply":"2022-11-10T14:45:08.302827Z"},"papermill":{"duration":0.026627,"end_time":"2022-11-10T14:45:08.306271","exception":false,"start_time":"2022-11-10T14:45:08.279644","status":"completed"},"tags":[],"id":"811225e4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if sampling == 'hybrid':\n    # 3) Hybrid approach","metadata":{"execution":{"iopub.execute_input":"2022-11-10T14:45:08.34141Z","iopub.status.busy":"2022-11-10T14:45:08.340534Z","iopub.status.idle":"2022-11-10T14:45:09.981732Z","shell.execute_reply":"2022-11-10T14:45:09.980286Z"},"papermill":{"duration":1.661906,"end_time":"2022-11-10T14:45:09.984509","exception":false,"start_time":"2022-11-10T14:45:08.322603","status":"completed"},"tags":[],"id":"03c4af34","outputId":"16a902a3-4d74-48b6-e9af-74a99dcbf530"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4. 학습 데이터 분할","metadata":{"id":"3f502828","papermill":{"duration":0.0169,"end_time":"2022-11-10T14:45:10.069019","exception":false,"start_time":"2022-11-10T14:45:10.052119","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# 첫번째 테스트용으로 사용하고, 실제 학습시에는 K-Fold CV를 사용합니다.\n# train : test = 8 : 2\nfrom sklearn.model_selection import train_test_split\n\nX = X_s\ny = y_s\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)\nprint(X_train.shape, X_val.shape, y_train.shape, y_val.shape)","metadata":{"execution":{"iopub.execute_input":"2022-11-10T14:45:10.105175Z","iopub.status.busy":"2022-11-10T14:45:10.104683Z","iopub.status.idle":"2022-11-10T14:45:10.134476Z","shell.execute_reply":"2022-11-10T14:45:10.132928Z"},"id":"a2ebefbd","papermill":{"duration":0.051073,"end_time":"2022-11-10T14:45:10.137727","exception":false,"start_time":"2022-11-10T14:45:10.086654","status":"completed"},"tags":[],"outputId":"ac9012f9-bf88-4c2d-9fb3-9cfa52429cbe"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# custom dataset & dataloader 정의\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 5. 학습 및 평가","metadata":{"id":"eb8950a4","papermill":{"duration":0.015979,"end_time":"2022-11-10T14:45:10.170437","exception":false,"start_time":"2022-11-10T14:45:10.154458","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# optimizer, criterion, lr_scheduler 정의\noptimizer = torch.optim.Adam(model.parameters(),\n                              lr=config['learning_rate'])\ncriterion = nn.MSELoss()\n#scheduler = ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data를 새롭게 representation하기 위한 AutoEncoder\nclass AutoEncoder(nn.Module):\n    def __init__(self):\n        super(AutoEncoder, self).__init__()\n    \n        self.fc1 = nn.Linear(8000, 4000, bias=False)\n        self.fc2 = nn.Linear(4000, 2000, bias=False)\n        self.fc3 = nn.Linear(2000, 1000, bias=False)\n        self.fc4 = nn.Linear(1000, 200, bias=True)\n        \n        self.defc1 = nn.Linear(200, 1000, bias=True)\n        self.defc2 = nn.Linear(1000, 2000, bias=False)\n        self.defc3 = nn.Linear(2000, 4000, bias=False)\n        self.defc4 = nn.Linear(4000, 8000, bias=False)\n\n        \n    def encoder(self, x):\n        x = self.fc1(x)\n        x = self.fc2(x)\n        x = self.fc3(x)\n        x = self.fc4(x)\n\n        return x\n    \n    def decoder(self, x):\n        x = self.defc1(x)\n        x = self.defc2(x)      \n        x = self.defc3(x)\n        x = self.defc4(x)\n\n        return x\n    \n    def forward(self, x):\n        \n        z = self.encoder(x)\n        x_hat = self.decoder(z)\n        \n        return x_hat, z","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_val_loss = 123456789.0   # worst value\ntol = 1e-8\npatience = 25\npatience_count = 0\n# Early stopping의 기준 : best_val_loss보다 tol 이하의 차이를 가지는 patience_count가 patience보다 같아지면 stop.\ntrain_losses = []\nval_losses = []\nepoch_end = 0\n\nfor epoch in tqdm(range(n_epochs)):\n    train_loss = 0.0\n    val_loss = 0.0\n\n    # training\n    model.train()  # training mode\n    for idx, data in enumerate(trainloader):\n        # dtype을 float32 -> float16으로 변환해서 메모리 사용량을 절반으로 줄이는 코드.\n        with torch.autocast(device_type='cuda', dtype=torch.float16):  ## tesla V100\n\n            X, y = data[0].to(device), data[1].to(device)\n            outputs = model(X)\n            loss = loss_fn(outputs, y)\n\n        optimizer.zero_grad()  # backprop할 때, gradient를 초기화해줌.\n        loss.backward()\n        optimizer.step()\n\n        train_loss = train_loss + loss.item()\n\n    scheduler.step()\n    train_loss = train_loss / len(trainloader)\n    train_losses.append(train_loss)\n    \n    if (epoch % 1000) == 0:\n        print(f\"Epoch {epoch} ---> Train loss : {train_loss:.4f}\")\n\n    # validation\n    model.eval()  # predict mode (dropout, batchnorm 같은 train/predict에서 다르게 동작하는 모듈을 전환해주는 함수.)\n    with torch.no_grad(): # locally disabling gradient computation\n        for idx, data in enumerate(validloader):\n            with torch.autocast(device_type='cuda', dtype=torch.float16):\n                X, y = data[0].to(device), data[1].to(device)\n                outputs = model(X)\n                loss = loss_fn(outputs, y)\n                val_loss = val_loss + loss.item()\n\n        val_loss = val_loss / len(validloader)\n        val_losses.append(val_loss)\n        \n        if (epoch % 1000) == 0:\n            print(f\"Epoch {epoch} ---> Validation loss : {val_loss:.6f}\")\n\n    # check best (val loss의 변화량이 tol보다 클 때)\n    if best_val_loss - val_loss > tol:\n        best_val_loss = val_loss\n        print(f\"Epoch {epoch} ---> (Best)Validation loss : {best_val_loss:.6f}\")\n        patience_count = 0\n    else:\n        patience_count = patience_count + 1\n\n    if patience_count > patience:\n        print(f\"Early stopping at {epoch} epoch.\")\n        epoch_end = epoch\n        break","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 7. 테스트 및 제출 파일 생성","metadata":{"id":"70b1e9a7","papermill":{"duration":0.020451,"end_time":"2022-11-10T16:07:45.148892","exception":false,"start_time":"2022-11-10T16:07:45.128441","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_num_part = pd.read_csv(base_path + 'test_numeric.csv.zip',\n                           nrows=1000)\ntest_date_part = pd.read_csv(base_path + 'test_date.csv.zip',\n                            nrows=1000)","metadata":{"execution":{"iopub.execute_input":"2022-11-10T16:07:45.392157Z","iopub.status.busy":"2022-11-10T16:07:45.391675Z","iopub.status.idle":"2022-11-10T16:07:45.845132Z","shell.execute_reply":"2022-11-10T16:07:45.843356Z"},"papermill":{"duration":0.479587,"end_time":"2022-11-10T16:07:45.84797","exception":false,"start_time":"2022-11-10T16:07:45.368383","status":"completed"},"tags":[],"id":"084526a4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## TO-DO\n## testloader를 사용해서, inference 코드 만들고 submission.csv 파일 완성하기!","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(base_path + \"sample_submission.csv.zip\")\nsubmission['Response'] = preds","metadata":{"execution":{"iopub.execute_input":"2022-11-10T16:10:35.65005Z","iopub.status.busy":"2022-11-10T16:10:35.649368Z","iopub.status.idle":"2022-11-10T16:10:35.907721Z","shell.execute_reply":"2022-11-10T16:10:35.906685Z"},"id":"dbceb995","papermill":{"duration":0.285324,"end_time":"2022-11-10T16:10:35.910473","exception":false,"start_time":"2022-11-10T16:10:35.625149","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.execute_input":"2022-11-10T16:10:35.959823Z","iopub.status.busy":"2022-11-10T16:10:35.959123Z","iopub.status.idle":"2022-11-10T16:10:37.17583Z","shell.execute_reply":"2022-11-10T16:10:37.174703Z"},"id":"8be4ba3f","papermill":{"duration":1.24429,"end_time":"2022-11-10T16:10:37.178722","exception":false,"start_time":"2022-11-10T16:10:35.934432","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Isolation Forest\n\n- anomaly detection의 대표적인 unsupervised learning 기법!","metadata":{"id":"uV0v6IEHFqp3"}},{"cell_type":"code","source":"from sklearn.ensemble import IsolationForest\n\niforest = IsolationForest(n_estimators=100, # 학습하는 트리 개수\n                          max_samples='auto', # min(256, n_samples), 트리별 학습 sample 수\n                          contamination='auto', # float 값을 넣으면 해당하는 %의 데이터를 outlier로 줌. e.g. 0.01 = 1%\n                          n_jobs=-1, # 모든 core를 사용\n                          random_state=42)\n#iforest.fit(X_train)\npreds = iforest.fit_predict(X_train) # fit & predict\npreds","metadata":{"id":"R1jVMRMJFqQR","outputId":"b0653cd9-9195-4d7c-aa29-9f0b957586cb"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" #iforest의 결과 -> 1 : normal, -1 : abnormal\n # 원본 데이터는 -> 0:normal, 1 : abnormal\npreds = pd.Series(preds).map({1 : 0, -1 : 1})\npreds","metadata":{"id":"E-BsZ4cHItFm","outputId":"81ca0714-006e-493a-b59c-c2ccfe2ce003"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds.value_counts()","metadata":{"id":"htVZ8H7QJQEL","outputId":"eb5d506b-687f-40d4-98ea-12ebb1dff899"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Train MCC : %.4f\" % evaluation_metric(y_train, preds))","metadata":{"id":"1T3NC_aiIcEK","outputId":"c6385371-ddd5-412e-c751-5373dd215c28"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"l-x1lO-dIqKV"},"execution_count":null,"outputs":[]}]}