{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### 학습 목표\n\n- 초대용량 데이터는 어떻게 처리할까요?\n\n\n- 데이터를 분할하여 불러오고 처리하는 방법에 대해서 연습해봅니다.\n\n\n- 불량 검출 문제에 대해 살펴봅니다.\n\n\n- Class imbalance 문제를 해결하는 방법에 대해 알아봅니다.","metadata":{}},{"cell_type":"markdown","source":"## Bosch Production Line Performance\n\n- 공정과정 데이터를 통해 제품 내부 불량 검출하기\n\n- 주어진 공정변수를 이용하여 정상/불량 분류하기","metadata":{"id":"U0IhpuicpE1o"}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\ntrain_cat_part = pd.read_csv(\"../input/bosch-production-line-performance/train_categorical.csv.zip\", nrows=1000)\ntrain_numeric_part = pd.read_csv(\"../input/bosch-production-line-performance/train_numeric.csv.zip\", nrows=1000)\ntrain_date_part = pd.read_csv(\"../input/bosch-production-line-performance/train_date.csv.zip\", nrows=1000)\n\nprint(train_numeric_part.shape, train_cat_part.shape, train_date_part.shape)\n","metadata":{"id":"improving-limitation","outputId":"874047c9-4bfc-47a0-ee85-24ceb5cf49e1","execution":{"iopub.status.busy":"2022-05-20T01:20:05.502283Z","iopub.execute_input":"2022-05-20T01:20:05.502765Z","iopub.status.idle":"2022-05-20T01:20:06.774844Z","shell.execute_reply.started":"2022-05-20T01:20:05.502680Z","shell.execute_reply":"2022-05-20T01:20:06.773343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_date_part # start_station, end_staion ","metadata":{"execution":{"iopub.status.busy":"2022-05-20T00:06:52.101719Z","iopub.execute_input":"2022-05-20T00:06:52.102025Z","iopub.status.idle":"2022-05-20T00:06:52.150073Z","shell.execute_reply.started":"2022-05-20T00:06:52.101992Z","shell.execute_reply":"2022-05-20T00:06:52.149357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### train_date 데이터를 요약해봅시다.\n- 각 row마다 측정이 처응된 station 번호와, 마지막으로 측정이 된 station번호를 찾습니다.","metadata":{}},{"cell_type":"code","source":"train_date_part","metadata":{"execution":{"iopub.status.busy":"2022-05-20T00:26:58.610214Z","iopub.execute_input":"2022-05-20T00:26:58.610539Z","iopub.status.idle":"2022-05-20T00:26:58.645847Z","shell.execute_reply.started":"2022-05-20T00:26:58.610506Z","shell.execute_reply":"2022-05-20T00:26:58.645024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"train_date_part.loc[0]","metadata":{"execution":{"iopub.status.busy":"2022-05-20T00:13:51.16031Z","iopub.execute_input":"2022-05-20T00:13:51.1606Z","iopub.status.idle":"2022-05-20T00:13:51.171513Z","shell.execute_reply.started":"2022-05-20T00:13:51.160571Z","shell.execute_reply":"2022-05-20T00:13:51.170554Z"}}},{"cell_type":"code","source":"row = train_date_part.loc[0]\nrow[row.index.str.contains(\"_S43_\")]# 0.01 == 6분\n\n# 1.station별 column 찾기 : 1157개 -> 52개\n# row.index.str.split('_', expand=True)\n\ncounts = train_date_part.drop(columns=\"Id\").count()\ndate_cols = counts.reset_index()[\"index\"].str.split(\"_\", expand=True)\ncol_idx = date_cols.drop_duplicates(1).index # 1번 열 기준으로 중복 제거 \ndate_cols = train_date_part.drop(columns=\"Id\").columns[col_idx]\ndate_cols","metadata":{"execution":{"iopub.status.busy":"2022-05-20T01:20:29.255523Z","iopub.execute_input":"2022-05-20T01:20:29.256092Z","iopub.status.idle":"2022-05-20T01:20:29.297725Z","shell.execute_reply.started":"2022-05-20T01:20:29.256058Z","shell.execute_reply":"2022-05-20T01:20:29.297089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#2. start_station, end_station 찾기\ntrain_date = pd.read_csv(\"../input/bosch-production-line-performance/train_date.csv.zip\", usecols=date_cols, nrows=1000)\n\n## 전체 데이터에서 column 순서대로 scan하면서 처음으로 NaN이 아닌 column의 station이 start_station,\n## 마지막까지 scan하면서 마지막으로 NaN이 아니었던 column의 station이 end_station\ntrain_date[\"start_station\"] = -1\ntrain_date[\"end_station\"] = -1\n\nfor col in train_date.drop(columns=\"Id\").columns:\n    nulls = train_date[col].isnotnull() # L0_S0_D1\n    station_name = col.split(\"_\")[1]\n    print(station_name)\n    display(notnulls)\n    \n    train_date[col].loc[notnulls & train_date[col].start , \"start_station\"] = station_name\n    \n    break\n","metadata":{"execution":{"iopub.status.busy":"2022-05-20T01:49:56.111409Z","iopub.execute_input":"2022-05-20T01:49:56.111916Z","iopub.status.idle":"2022-05-20T01:49:56.188877Z","shell.execute_reply.started":"2022-05-20T01:49:56.111882Z","shell.execute_reply":"2022-05-20T01:49:56.187915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_numeric_part = pd.read_csv(\"../input/bosch-production-line-performance/train_numeric.csv.zip\", usecols=[\"Id\", \"Response\"])\n#train_numeric_part","metadata":{"execution":{"iopub.status.busy":"2022-05-18T08:24:53.324921Z","iopub.execute_input":"2022-05-18T08:24:53.325195Z","iopub.status.idle":"2022-05-18T08:25:36.614514Z","shell.execute_reply.started":"2022-05-18T08:24:53.325167Z","shell.execute_reply":"2022-05-18T08:25:36.613266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_numeric_part","metadata":{"execution":{"iopub.status.busy":"2022-05-18T08:21:57.729508Z","iopub.execute_input":"2022-05-18T08:21:57.729789Z","iopub.status.idle":"2022-05-18T08:21:57.764347Z","shell.execute_reply.started":"2022-05-18T08:21:57.72976Z","shell.execute_reply":"2022-05-18T08:21:57.763408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_numeric_part.Response.value_counts(normalize=True)\nsns.countplot(data=train_numeric_part, y=\"Response\") # Target Value Distribution\n## Class imbalance problem, 클래스 불균형 문제!","metadata":{"execution":{"iopub.status.busy":"2022-05-19T04:08:31.340405Z","iopub.execute_input":"2022-05-19T04:08:31.340738Z","iopub.status.idle":"2022-05-19T04:08:31.504746Z","shell.execute_reply.started":"2022-05-19T04:08:31.340701Z","shell.execute_reply":"2022-05-19T04:08:31.503829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_numeric_part.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-05-19T04:08:34.762939Z","iopub.execute_input":"2022-05-19T04:08:34.763473Z","iopub.status.idle":"2022-05-19T04:08:34.801505Z","shell.execute_reply.started":"2022-05-19T04:08:34.763424Z","shell.execute_reply":"2022-05-19T04:08:34.800346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1. 대용량 데이터 다루기\n\nTechnique1. 잘라서 불러오기\n\n\nTechnique2. Reducing Memory Technique (dtype 변경하기)shape","metadata":{}},{"cell_type":"code","source":"missing_ratio = pd.Series(index=train_numeric_part.columns, data=np.zeros(len(train_numeric_part.columns))) #970\ncolumn_means = pd.Series(index=train_numeric_part.columns, data=np.zeros(len(train_numeric_part.columns))) #970\n\ndisplay(missing_ratio)","metadata":{"execution":{"iopub.status.busy":"2022-05-19T04:08:39.351262Z","iopub.execute_input":"2022-05-19T04:08:39.351559Z","iopub.status.idle":"2022-05-19T04:08:39.361712Z","shell.execute_reply.started":"2022-05-19T04:08:39.351523Z","shell.execute_reply":"2022-05-19T04:08:39.360732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# io chunk 사용\n\n#### 대용량 데이터가 메인 메모리에 한번에 적재가 안될때.\n1. 사용할 컬럼을 몇개 불러온다 (usecols)\n2. 데이터를 쪼개서 임시로 불러와서 계산하고, 다시 불러온다.(chunksize)","metadata":{}},{"cell_type":"code","source":"# Technique 1.\nlength = 0\nfor chunk in pd.read_csv(\"../input/bosch-production-line-performance/train_numeric.csv.zip\", \n                         \n                         chunksize=100000):\n    #display(chunk)\n    missing_ratio = missing_ratio + chunk.isnull().sum() # 결측치 합\n    temp2 = chunk.sum()\n    length = length + len(chunk)\n    column_means = column_means + temp2 # 누적 평균\n    #break\n\ndisplay(missing_ratio)\ndisplay(column_means/length)","metadata":{"id":"isolated-barbados","execution":{"iopub.status.busy":"2022-05-19T04:08:50.530184Z","iopub.execute_input":"2022-05-19T04:08:50.53097Z","iopub.status.idle":"2022-05-19T04:10:54.367482Z","shell.execute_reply.started":"2022-05-19T04:08:50.53093Z","shell.execute_reply":"2022-05-19T04:10:54.366435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 차원의 저주(Curse of Dimensioality)\n- 머신러닝 모델이 데이터가 존재하는 공간이 너무 고차원 일 때, feature 가 너무 많을 때,\n- 머신러닝 모델이 예측력을 읽어버리는 문제(성능이 저하)\n---\n- 해결책\n> 차원을 줄입니다!(=column 수를 줄인다.)\n> 1. feature selection : 분석에 사용할 column을 직접 고른다.\n>> 결측치가 많은 column을 버리거나, 도메인 지식으로 필요한 변수를 고르거나, ....\n> 2. feature extraction : dimensionality reduction method(e.g. PCA)를 사용한다.","metadata":{"execution":{"iopub.status.busy":"2022-05-19T04:11:59.944443Z","iopub.execute_input":"2022-05-19T04:11:59.945457Z","iopub.status.idle":"2022-05-19T04:11:59.952668Z","shell.execute_reply.started":"2022-05-19T04:11:59.945405Z","shell.execute_reply":"2022-05-19T04:11:59.951613Z"}}},{"cell_type":"code","source":"missing_ratio = missing_ratio / length","metadata":{"execution":{"iopub.status.busy":"2022-05-19T04:19:10.86029Z","iopub.execute_input":"2022-05-19T04:19:10.861015Z","iopub.status.idle":"2022-05-19T04:19:10.86549Z","shell.execute_reply.started":"2022-05-19T04:19:10.860961Z","shell.execute_reply":"2022-05-19T04:19:10.864823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 결측치가 50%를 넘는 column을 삭제\nmissing_ratio > 0.5\ntrain_numeric_part.drop(columns=train_numeric_part.columns[missing_ratio > 0.5]) # 일반적\n","metadata":{"execution":{"iopub.status.busy":"2022-05-19T04:25:32.406632Z","iopub.execute_input":"2022-05-19T04:25:32.406931Z","iopub.status.idle":"2022-05-19T04:25:32.453651Z","shell.execute_reply.started":"2022-05-19T04:25:32.406899Z","shell.execute_reply":"2022-05-19T04:25:32.452966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 결측비율이 50%를 넘지 않는 column들만 선택\n\nuse_cols = train_numeric_part.columns[missing_ratio <= 0.5]","metadata":{"execution":{"iopub.status.busy":"2022-05-19T04:42:23.171748Z","iopub.execute_input":"2022-05-19T04:42:23.172278Z","iopub.status.idle":"2022-05-19T04:42:23.179372Z","shell.execute_reply.started":"2022-05-19T04:42:23.172214Z","shell.execute_reply":"2022-05-19T04:42:23.177991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_numeric_part2 = pd.read_csv(\"../input/bosch-production-line-performance/train_numeric.csv.zip\", usecols=[\"Id\", \"Response\"])\ntrain_numeric_part2","metadata":{"execution":{"iopub.status.busy":"2022-05-19T04:35:31.356326Z","iopub.execute_input":"2022-05-19T04:35:31.356667Z","iopub.status.idle":"2022-05-19T04:36:03.506115Z","shell.execute_reply.started":"2022-05-19T04:35:31.356631Z","shell.execute_reply":"2022-05-19T04:36:03.505253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.decomposition import PCA\ndata = pd.read_csv(\"../input/bosch-production-line-performance/train_numeric.csv.zip\",\n                        usecols=use_cols)\ndata = data.fillna(0) # 결측치를 0으로 채움\nids = data.Id\nresponses = data.Response\ndata = data.drop(columns=[\"Id\", \"Response\"])\npca = PCA(n_components=6)\nX = pca.fit_transform(data)\npca_df = pd.DataFrame(columns=[f\"PC{i}\" for i in range(1, 7)],\n                     data=X)\ndisplay(pca_df)","metadata":{"execution":{"iopub.status.busy":"2022-05-19T06:34:20.878345Z","iopub.execute_input":"2022-05-19T06:34:20.878976Z","iopub.status.idle":"2022-05-19T06:35:36.984608Z","shell.execute_reply.started":"2022-05-19T06:34:20.878934Z","shell.execute_reply":"2022-05-19T06:35:36.983639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nfor chunk in pd.read_csv(\"../input/bosch-production-line-performance/train_numeric.csv.zip\", \n                         usecols=use_cols, chunksize=10000):\n    \n    chunk = chunk.fillna(0) # 0 fill\n    #display(chunk)\n    temp = chunk.drop(columns=[\"Id\", \"Response\"])\n    #machine learning using PCA\n    pca = PCA(n_components=15)\n    X = pca.fit_transform(temp)\n    pca_df = pd.DataFrame(columns=[f\"PC{i}\" for i in range(1,16) ], data=X)\n    display(pca_df)\n    break","metadata":{"execution":{"iopub.status.busy":"2022-05-19T04:50:36.729514Z","iopub.execute_input":"2022-05-19T04:50:36.72984Z","iopub.status.idle":"2022-05-19T04:50:37.500436Z","shell.execute_reply.started":"2022-05-19T04:50:36.729806Z","shell.execute_reply":"2022-05-19T04:50:37.499101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_numeric_part2.info()","metadata":{"execution":{"iopub.status.busy":"2022-05-19T04:36:15.921889Z","iopub.execute_input":"2022-05-19T04:36:15.922438Z","iopub.status.idle":"2022-05-19T04:36:15.945267Z","shell.execute_reply.started":"2022-05-19T04:36:15.9223Z","shell.execute_reply":"2022-05-19T04:36:15.944472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### dtype\n- np.int8, np.int16, np.int32, np.int64 : 정수 하나를 몇 bit로 표현할 것인가?\n> int8 : -2^7 ~ 2^your 7-1 --> -128 ~ 127 --> 0000000000 ~ 11111111\n- np.float32 / np.float64 : 실수 하나를 몇 bit로 표현할 것인가?\n> 실수 표현은 항상 근처를 쓰기 때문에, 얼마나 자세하게 표현할지의 차이!\n\n> 필연적으로 발생하는 수치적 차이를 \"numerical error\"라고 부릅니다.","metadata":{}},{"cell_type":"code","source":"# Technique 2. dtype 변경\npca_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-05-19T05:42:43.223605Z","iopub.execute_input":"2022-05-19T05:42:43.223894Z","iopub.status.idle":"2022-05-19T05:42:43.242282Z","shell.execute_reply.started":"2022-05-19T05:42:43.223863Z","shell.execute_reply":"2022-05-19T05:42:43.241158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca_df.astype(np.float32).info()","metadata":{"execution":{"iopub.status.busy":"2022-05-19T05:43:22.32681Z","iopub.execute_input":"2022-05-19T05:43:22.327106Z","iopub.status.idle":"2022-05-19T05:43:22.342485Z","shell.execute_reply.started":"2022-05-19T05:43:22.327073Z","shell.execute_reply":"2022-05-19T05:43:22.34181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2. 불량 검출 문제 다루기 (Class Imbalance problem)\n\nTechnique 1. Sampling Method (Under / Over)\n> binary classification (이진 분류) : 정상 혹은 불량 로 예측하는 문제!\n\n> 일반적으로 불량에 해당하는 케이스가 매우 매우 적습니다. (minority class)\n\n> 일반적으로 정산에 해당하는 케이스가 아주아주 많습니다. (majority class)\n\n> 머신러닝 분류 문제에서는 클래스의 비율이 비슷할 수록 학습이 잘됩니다.\n\n>> minority -> majority (oversampling) \n\n>> majority -> minority (undersampling)\n\n---\nTechnique 2. Change to Outlier Detection problem\n\n- 주류(또는 대다수) : majority\n\n- 주류가 아닌 데이터를 outlier(이상치)라고 판단하는 방법.\n\n- IQR, IsolationForest (ML) , DBCAN (ML)\n\n\n(Avanced) Technique 3. Semi-supervised Learing (DL), MakinaRocks","metadata":{}},{"cell_type":"code","source":"# Technique 1.\n# X : feature vector (독립변수), 학습 데이터(문제)\n# y : target value (종속변수), 목표값 (정답)\nX = pca_df\ny = responses\nprint(X.shape, y.shape)\n\n## 1-1. Undersampling - y에서 1인 개수로 0인 데이터를 줄이는 것.\nnormal = X[y==0]\nfailure = X[y==1]\nprint(normal.shape, failure.shape) # 1176 --> 6879\n\nnormal_under = normal.sample(n=failure.shape[0], random_state=0xC0FFEE)\nprint(normal_under.shape, failure.shape)","metadata":{"execution":{"iopub.status.busy":"2022-05-19T06:48:15.954728Z","iopub.execute_input":"2022-05-19T06:48:15.955165Z","iopub.status.idle":"2022-05-19T06:48:16.121083Z","shell.execute_reply.started":"2022-05-19T06:48:15.955131Z","shell.execute_reply":"2022-05-19T06:48:16.119653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 1-2 Oversampling - yr가 1인 데이터를 y가 0인 데이터의 개수로 늘리는 것.\nfrom imblearn.over_sampling import SMOTE\n\nsm = SMOTE()\nX_oversampled, y_oversampled = sm.fit_resample(X,y)\nprint(X_oversampled.shape, y_oversampled.shape)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-19T06:59:49.113527Z","iopub.execute_input":"2022-05-19T06:59:49.114058Z","iopub.status.idle":"2022-05-19T06:59:49.885191Z","shell.execute_reply.started":"2022-05-19T06:59:49.114004Z","shell.execute_reply":"2022-05-19T06:59:49.884161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 1-3 . \n\n### 정상데이터를 500,000개로 undersampling 한뒤, oversamling을 수행하여\n### 데이터를 1,000,000객 되도록 만들어주세요!\nnormal_hybrid = normal.sample(n=500000, random_state=42)\nprint(normal_hybrid.shape)\n#X_hybrid, y_hybrid = ","metadata":{"execution":{"iopub.status.busy":"2022-05-19T07:50:20.477179Z","iopub.execute_input":"2022-05-19T07:50:20.477507Z","iopub.status.idle":"2022-05-19T07:50:20.581654Z","shell.execute_reply.started":"2022-05-19T07:50:20.477473Z","shell.execute_reply":"2022-05-19T07:50:20.580676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y[normal_hybrid.index].shape","metadata":{"execution":{"iopub.status.busy":"2022-05-19T07:47:49.3752Z","iopub.execute_input":"2022-05-19T07:47:49.375755Z","iopub.status.idle":"2022-05-19T07:47:49.419528Z","shell.execute_reply.started":"2022-05-19T07:47:49.375716Z","shell.execute_reply":"2022-05-19T07:47:49.418357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 정상 불량을 모두 넣어야 하는데\nsm = SMOTE()\nX_hybrid, y_hybrid = sm.fit_resample(normal_hybrid,y[normal_hybrid.index])\nprint(X_hybrid.shape, y_hybrid.shape)","metadata":{"execution":{"iopub.status.busy":"2022-05-19T07:48:38.310114Z","iopub.execute_input":"2022-05-19T07:48:38.310434Z","iopub.status.idle":"2022-05-19T07:48:38.426419Z","shell.execute_reply.started":"2022-05-19T07:48:38.310399Z","shell.execute_reply":"2022-05-19T07:48:38.424994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_oversampled","metadata":{"execution":{"iopub.status.busy":"2022-05-19T07:01:55.821433Z","iopub.execute_input":"2022-05-19T07:01:55.821999Z","iopub.status.idle":"2022-05-19T07:01:55.840923Z","shell.execute_reply.started":"2022-05-19T07:01:55.821949Z","shell.execute_reply":"2022-05-19T07:01:55.839764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"normal.shape[0]*2","metadata":{"execution":{"iopub.status.busy":"2022-05-19T07:00:29.191618Z","iopub.execute_input":"2022-05-19T07:00:29.192211Z","iopub.status.idle":"2022-05-19T07:00:29.198456Z","shell.execute_reply.started":"2022-05-19T07:00:29.192173Z","shell.execute_reply":"2022-05-19T07:00:29.197512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 1-3. Hybrid Approach : Undersampling + Oversampling\n\n### 정상 데이터를 500,000개로 undersampling한 뒤, 불량 데이터를 oversampling을 수행하여\n### 데이터를 1,000,000개가 되도록 만들어주세요!!\n\nX_under = normal.sample(n=500000).reset_index(drop=True)\ny_under = np.zeros(len(X_under))\nX_total = pd.concat([X_under, failure]).reset_index(drop=True)\ny_total = np.hstack([y_under, np.ones(len(failure))]).astype(int)\n\n\n# SMOTE에 정상과 불량 데이터가 합쳐진 전체 데이터를 넣으면 알아서,\n# majority class 크기에 맞게 oversampling이 됩니다.\nX_hybrid, y_hybrid = sm.fit_resample(X_total, y_total)\nprint(X_hybrid.shape, y_hybrid.shape)","metadata":{"execution":{"iopub.status.busy":"2022-05-19T08:08:36.527992Z","iopub.execute_input":"2022-05-19T08:08:36.528344Z","iopub.status.idle":"2022-05-19T08:08:37.423597Z","shell.execute_reply.started":"2022-05-19T08:08:36.52831Z","shell.execute_reply":"2022-05-19T08:08:37.422668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Technique 2.\nsns.boxplot(train_numeric_part.L0_S0_F0)","metadata":{"execution":{"iopub.status.busy":"2022-05-19T08:20:22.961654Z","iopub.execute_input":"2022-05-19T08:20:22.961943Z","iopub.status.idle":"2022-05-19T08:20:23.158992Z","shell.execute_reply.started":"2022-05-19T08:20:22.961912Z","shell.execute_reply":"2022-05-19T08:20:23.157934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def IQR_outlier(data, column):\n    temp = data[column].dropna() # 결측치가 없어야 함.\n    \n    q25 = np.percentile(temp, 25) \n    q75 = np.percentile(temp, 75)\n    \n    IQR = q75 - q25 # Inter-Quartile Range\n    lower_bound = q25 - IQR * 1.5\n    upper_bound = q75 + IQR * 1.5\n    \n    outlier = temp.loc[(temp < lower_bound) | (temp > upper_bound)]\n    return outlier\n        ","metadata":{"execution":{"iopub.status.busy":"2022-05-19T08:39:39.130256Z","iopub.execute_input":"2022-05-19T08:39:39.130583Z","iopub.status.idle":"2022-05-19T08:39:39.139012Z","shell.execute_reply.started":"2022-05-19T08:39:39.130536Z","shell.execute_reply":"2022-05-19T08:39:39.138355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IQR_outlier(train_numeric_part, \"L0_S0_F0\")","metadata":{"execution":{"iopub.status.busy":"2022-05-19T08:39:41.013469Z","iopub.execute_input":"2022-05-19T08:39:41.013922Z","iopub.status.idle":"2022-05-19T08:39:41.027262Z","shell.execute_reply.started":"2022-05-19T08:39:41.01387Z","shell.execute_reply":"2022-05-19T08:39:41.025884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}