{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### 학습 목표\n\n- 초대용량 데이터는 어떻게 처리할까요?\n\n\n- 데이터를 분할하여 불러오고 처리하는 방법에 대해서 연습해봅니다.\n\n\n- 불량 검출 문제에 대해 살펴봅니다.\n\n\n- Class imbalance 문제를 해결하는 방법에 대해 알아봅니다.","metadata":{}},{"cell_type":"markdown","source":"## Bosch Production Line Performance\n\n- 공정과정 데이터를 통해 제품 내부 불량 검출하기\n\n- 주어진 공정변수를 이용하여 정상/불량 분류하기","metadata":{"id":"U0IhpuicpE1o"}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\ntrain_numeric_part = pd.read_csv('../input/bosch-production-line-performance/train_numeric.csv.zip',\n                                nrows=1000)     # 데이터를 불러올 때 몇개의 Row를 불러올것인가\ntrain_categorical_part = pd.read_csv('../input/bosch-production-line-performance/train_categorical.csv.zip',\n                                    nrows=1000)\ntrain_date_part = pd.read_csv('../input/bosch-production-line-performance/train_date.csv.zip',\n                             nrows=1000)\n\nprint(train_numeric_part.shape, train_categorical_part.shape, train_date_part.shape)","metadata":{"id":"improving-limitation","outputId":"874047c9-4bfc-47a0-ee85-24ceb5cf49e1","execution":{"iopub.status.busy":"2022-06-30T07:02:52.161865Z","iopub.execute_input":"2022-06-30T07:02:52.162264Z","iopub.status.idle":"2022-06-30T07:02:53.913106Z","shell.execute_reply.started":"2022-06-30T07:02:52.162176Z","shell.execute_reply":"2022-06-30T07:02:53.911995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_date_part","metadata":{"execution":{"iopub.status.busy":"2022-06-30T07:02:53.914842Z","iopub.execute_input":"2022-06-30T07:02:53.915171Z","iopub.status.idle":"2022-06-30T07:02:53.958128Z","shell.execute_reply.started":"2022-06-30T07:02:53.915141Z","shell.execute_reply":"2022-06-30T07:02:53.957017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1. 대용량 데이터 다루기\n\n**Technique1. 잘라서 불러오기 (read_csv 함수를 사용하는 방법)**\n\n1) row 단위로 일부를 가져오기\n\n- nrows : 지정한 row 개수만큼만 잘라서 가져옵니다. (위에서부터)\n\n- chunksize : for문을 사용해서 매번 특정 row개수씩 가져옵니다. (위에서부터 순서대로)\n\n\n\n2) column 단위로 일부를 가져오기\n\n- usecols : 사용할 column들만 지정해서 가져옵니다.\n\n\n\n\n\n**Technique2. Reducing Memory Technique (dtype 변경하기)**","metadata":{}},{"cell_type":"code","source":"# Technique 1.\n## 1) train_date_part를 가지고, 각 station별 feature를 뽑아봅니다.\n### 가정. station별로 측정 시간이 모두 같으면, 하나씩만 사용하자. (X)\n### 결론. station별 측정 시간을 평균값으로 사용하자. (time feature)\nstation_names = train_date_part.count().reset_index()['index'].str.split(\"_\", expand=True)[1][1:].drop_duplicates().values\ntrain_date = pd.DataFrame()\n# 10만개씩 가져옴.\nchunks = pd.read_csv('../input/bosch-production-line-performance/train_date.csv.zip', chunksize=100000)\nfor chunk in chunks:\n    print(chunk.shape)\n    temp_date = pd.DataFrame()\n\n    for station in station_names:\n        station_cols = chunk.columns[chunk.columns.str.contains(station)]\n        col_means = chunk[station_cols].mean(axis=1) # 각 station에 대해서 row별 평균\n        temp_date[station] = col_means\n    train_date = pd.concat([train_date, temp_date]) ## 각 station별 측정시간의 평균으로 데이터를 요약한 정보.\n\ntrain_date","metadata":{"id":"isolated-barbados","execution":{"iopub.status.busy":"2022-06-30T07:02:53.959987Z","iopub.execute_input":"2022-06-30T07:02:53.960722Z","iopub.status.idle":"2022-06-30T07:05:35.660542Z","shell.execute_reply.started":"2022-06-30T07:02:53.960686Z","shell.execute_reply":"2022-06-30T07:05:35.659194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_date.info()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T07:05:35.663426Z","iopub.execute_input":"2022-06-30T07:05:35.663751Z","iopub.status.idle":"2022-06-30T07:05:35.829851Z","shell.execute_reply.started":"2022-06-30T07:05:35.663723Z","shell.execute_reply":"2022-06-30T07:05:35.828647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## TO-DO\n# train_numeric 데이터를 불러와서, 전체 결측치 비율을 column별로 측정한 다음에\n# 결측치가 50%를 넘는 모든 column을 제거한 DataFrame을 train_numeric으로 저장해서 출력해주세요.\n\nchunks = pd.read_csv('../input/bosch-production-line-performance/train_numeric.csv.zip',\n                    chunksize=100000)\n\n# column별 결측치를 담을 변수.(pd.Series)\nmissing_ratio = pd.Series(index=train_numeric_part.columns,\n                         data=np.zeros(len(train_numeric_part.columns)))\n# 전체 데이터 개수\nn_data = 0\n\nfor chunk in chunks:\n    temp = chunk.isnull().sum() # column별 결측치\n    \n    missing_ratio = missing_ratio + temp\n    n_data = n_data + len(chunk)\n    \nmissing_ratio = missing_ratio / n_data # 합 -> 비율 변경\nmissing_ratio","metadata":{"execution":{"iopub.status.busy":"2022-06-30T07:05:35.831950Z","iopub.execute_input":"2022-06-30T07:05:35.832397Z","iopub.status.idle":"2022-06-30T07:06:59.776698Z","shell.execute_reply.started":"2022-06-30T07:05:35.832356Z","shell.execute_reply":"2022-06-30T07:06:59.775304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_cols = train_numeric_part.columns[missing_ratio >= 0.5]\nusecols = np.setdiff1d(train_numeric_part.columns, drop_cols) # 158 columns\n\ntrain_numeric = pd.read_csv('../input/bosch-production-line-performance/train_numeric.csv.zip',\n                           usecols=usecols)\ntrain_numeric","metadata":{"execution":{"iopub.status.busy":"2022-06-30T07:06:59.778352Z","iopub.execute_input":"2022-06-30T07:06:59.778673Z","iopub.status.idle":"2022-06-30T07:07:42.042392Z","shell.execute_reply.started":"2022-06-30T07:06:59.778647Z","shell.execute_reply":"2022-06-30T07:07:42.041285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n\ngc.collect() # 메모리 청소기","metadata":{"execution":{"iopub.status.busy":"2022-06-30T07:07:42.043945Z","iopub.execute_input":"2022-06-30T07:07:42.044925Z","iopub.status.idle":"2022-06-30T07:07:42.219868Z","shell.execute_reply.started":"2022-06-30T07:07:42.044879Z","shell.execute_reply":"2022-06-30T07:07:42.218439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# JOIN 연산을 할 기준인 Id column을 추가합니다.\ntrain_date.columns = 't_' + train_date.columns  # 시간 관련 feature임을 알리기 위해 column name 변경.\ntrain_date[\"Id\"] = train_numeric.Id  # Id column 추가.\ntrain_date","metadata":{"execution":{"iopub.status.busy":"2022-06-30T07:07:42.221385Z","iopub.execute_input":"2022-06-30T07:07:42.222220Z","iopub.status.idle":"2022-06-30T07:07:42.480251Z","shell.execute_reply.started":"2022-06-30T07:07:42.222180Z","shell.execute_reply":"2022-06-30T07:07:42.479227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Id column을 기준으로 같은 Id를 가지는 데이터끼리 column을 합쳐줍니다. ---> INNER JOIN\ntrain = pd.merge(train_numeric, train_date, on=\"Id\")\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-06-30T07:07:42.481594Z","iopub.execute_input":"2022-06-30T07:07:42.481880Z","iopub.status.idle":"2022-06-30T07:07:44.580931Z","shell.execute_reply.started":"2022-06-30T07:07:42.481854Z","shell.execute_reply":"2022-06-30T07:07:44.579651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 안쓰는 데이터 제거 (메모리에서 제거)\ndel train_numeric\ndel train_date\ndel train_numeric_part\ndel train_date_part\ndel train_categorical_part\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T07:07:44.584055Z","iopub.execute_input":"2022-06-30T07:07:44.584393Z","iopub.status.idle":"2022-06-30T07:07:44.701852Z","shell.execute_reply.started":"2022-06-30T07:07:44.584366Z","shell.execute_reply":"2022-06-30T07:07:44.700612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 결측치가 50%가 넘는 column들 제거.\n#drop_cols = train.columns[train.isnull().mean() >= 0.5]\nusecols = train.columns[train.isnull().mean() < 0.5]\ntrain = train[usecols]\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-06-30T07:07:44.703487Z","iopub.execute_input":"2022-06-30T07:07:44.703949Z","iopub.status.idle":"2022-06-30T07:07:48.935038Z","shell.execute_reply.started":"2022-06-30T07:07:44.703907Z","shell.execute_reply":"2022-06-30T07:07:48.933799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 결측치를 모두 0으로 채웁니다.\ntrain = train.fillna(0)\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-06-30T07:07:48.936434Z","iopub.execute_input":"2022-06-30T07:07:48.936753Z","iopub.status.idle":"2022-06-30T07:07:50.701671Z","shell.execute_reply.started":"2022-06-30T07:07:48.936725Z","shell.execute_reply":"2022-06-30T07:07:50.700578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T07:07:50.702962Z","iopub.execute_input":"2022-06-30T07:07:50.703272Z","iopub.status.idle":"2022-06-30T07:07:50.820131Z","shell.execute_reply.started":"2022-06-30T07:07:50.703245Z","shell.execute_reply":"2022-06-30T07:07:50.818653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Technique 2.\ndef reduce_mem_usage(props):\n    start_mem_usg = props.memory_usage().sum() / 1024**2 \n    print(\"Memory usage of properties dataframe is :\",start_mem_usg,\" MB\")\n    NAlist = [] # Keeps track of columns that have missing values filled in. \n    for col in props.columns:\n        if props[col].dtype != object:  # Exclude strings\n            \n            # Print current column type\n#             print(\"******************************\")\n#             print(\"Column: \",col)\n#             print(\"dtype before: \",props[col].dtype)\n            \n            # make variables for Int, max and min\n            IsInt = False\n            mx = props[col].max()\n            mn = props[col].min()\n            \n            # Integer does not support NA, therefore, NA needs to be filled\n            if not np.isfinite(props[col]).all(): \n                NAlist.append(col)\n                props[col].fillna(mn-1,inplace=True)  \n                   \n            # test if column can be converted to an integer\n            asint = props[col].fillna(0).astype(np.int64)\n            result = (props[col] - asint)\n            result = result.sum()\n            if result > -0.01 and result < 0.01:\n                IsInt = True\n                \n            # Make Integer/unsigned Integer datatypes\n            if IsInt:\n                if mn >= 0:\n                    if mx < 255:\n                        props[col] = props[col].astype(np.uint8)\n                    elif mx < 65535:\n                        props[col] = props[col].astype(np.uint16)\n                    elif mx < 4294967295:\n                        props[col] = props[col].astype(np.uint32)\n                    else:\n                        props[col] = props[col].astype(np.uint64)\n                else:\n                    if mn > np.iinfo(np.int8).min and mx < np.iinfo(np.int8).max:\n                        props[col] = props[col].astype(np.int8)\n                    elif mn > np.iinfo(np.int16).min and mx < np.iinfo(np.int16).max:\n                        props[col] = props[col].astype(np.int16)\n                    elif mn > np.iinfo(np.int32).min and mx < np.iinfo(np.int32).max:\n                        props[col] = props[col].astype(np.int32)\n                    elif mn > np.iinfo(np.int64).min and mx < np.iinfo(np.int64).max:\n                        props[col] = props[col].astype(np.int64)    \n            \n            # Make float datatypes 32 bit\n            else:\n                props[col] = props[col].astype(np.float32)\n            \n            # Print new column type\n#             print(\"dtype after: \",props[col].dtype) \n#             print(\"******************************\")\n    \n    # Print final result\n    print(\"___MEMORY USAGE AFTER COMPLETION:___\")\n    mem_usg = props.memory_usage().sum() / 1024**2 \n    print(\"Memory usage is: \",mem_usg,\" MB\")\n    print(\"This is \",100*mem_usg/start_mem_usg,\"% of the initial size\")\n    return props, NAlist","metadata":{"execution":{"iopub.status.busy":"2022-06-30T07:07:50.821477Z","iopub.execute_input":"2022-06-30T07:07:50.821869Z","iopub.status.idle":"2022-06-30T07:07:50.838591Z","shell.execute_reply.started":"2022-06-30T07:07:50.821814Z","shell.execute_reply":"2022-06-30T07:07:50.837692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train, _ = reduce_mem_usage(train)","metadata":{"execution":{"iopub.status.busy":"2022-06-30T07:07:50.839674Z","iopub.execute_input":"2022-06-30T07:07:50.840980Z","iopub.status.idle":"2022-06-30T07:08:47.014992Z","shell.execute_reply.started":"2022-06-30T07:07:50.840934Z","shell.execute_reply":"2022-06-30T07:08:47.013805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T07:08:47.016463Z","iopub.execute_input":"2022-06-30T07:08:47.017678Z","iopub.status.idle":"2022-06-30T07:08:47.031154Z","shell.execute_reply.started":"2022-06-30T07:08:47.017629Z","shell.execute_reply":"2022-06-30T07:08:47.030242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2. 불량 검출 문제 다루기 (Class Imbalance problem)\n\n#### Technique 1. Sampling Method (Under / Over)\n\n\n- 1000000(정상) : 100(불량) = 10000 : 1   ----> 컴퓨터는 모든 데이터를 정상이라고 예측하려고 합니다.\n\n--> 1:1 비율로 변경이 필요합니다! (예측 성능이 좋아짐)\n\n\n**개수가 많은 클래스 = majority class, 개수가 적은 클래스 = minority class**\n\n\n**1) Undersampling**\n- majority class에서 sampling을 하여, minority class의 수로 맞춰주는 기법.\n\ne.g. 1000000->100   ------> 100 : 100 = 1 : 1\n\n\n**2) Oversampling**\n- minority class에서 sampling(generation)을 하여, majority class의 수를 맞춰주는 기법.\n\ne.g. 100->1000000 --------> 1000000 : 1000000 = 1 : 1 \n\n\n**3) Hybrid approach (under + over)**\n- undersampling과 oversampling을 사용하여 1:1의 비율을 맞춰주는 기법.\n\ne.g. 1000000->10000, 100->10000  -------> 10000 : 10000 = 1 : 1\n\n\n\n\n#### Technique 2. Change to Outlier Detection problem  (SKIP)\n\n- DBSCAN, IsolationForest (Unsupervised learning approach)","metadata":{}},{"cell_type":"code","source":"# Technique 1.\nsns.countplot(data=train, x=\"Response\")\ntrain.Response.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T07:08:47.032430Z","iopub.execute_input":"2022-06-30T07:08:47.033361Z","iopub.status.idle":"2022-06-30T07:08:47.324707Z","shell.execute_reply.started":"2022-06-30T07:08:47.033327Z","shell.execute_reply":"2022-06-30T07:08:47.323709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"normal = train[train.Response == 0]\nabnormal = train[train.Response == 1]\n\n\n# Undersampling\nunder_normal = normal.sample(n=len(abnormal), random_state=42)\nu_train = pd.concat([under_normal, abnormal])\nu_train # 13758 = 6879 x 2","metadata":{"execution":{"iopub.status.busy":"2022-06-30T07:23:40.328707Z","iopub.execute_input":"2022-06-30T07:23:40.329674Z","iopub.status.idle":"2022-06-30T07:23:41.277508Z","shell.execute_reply.started":"2022-06-30T07:23:40.329629Z","shell.execute_reply":"2022-06-30T07:23:41.276327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Oversampling (= Generation)\nfrom imblearn.over_sampling import SMOTE\n\nX_train = train.drop(columns=[\"Id\", 'Response'])\ny_train = train.Response\n\n## fit_resample 함수가 자동으로 y_train을 보고 minority class를 찾아서 1:1의 비율로 맞춰줍니다.\noX_train, oy_train = SMOTE().fit_resample(X_train, y_train)\nprint(oX_train.shape, oy_train.shape) # (1176868, 168) / (1176868, 168)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Hybrid (under + over)\nfrom imblearn.over_sampling import SMOTE\nn_sample = 10000\n\nnormal = train[train.Response == 0].sample(n=n_sample) # majority class에서 10,000개 Undersampling\nabnormal = train[train.Response == 1]\n\ntemp = pd.concat([normal, abnormal]) # 10000 + 6879\nX_temp = temp.drop(columns=[\"Id\", 'Response'])\ny_temp = temp.Response\n\nhX_train, hy_train = SMOTE().fit_resample(X_temp, y_temp) # 10000 + 10000(6879 -> 10000)\nprint(hX_train.shape, hy_train.shape) # (20000, 168) / (20000, 168)","metadata":{"execution":{"iopub.status.busy":"2022-06-30T07:39:26.918516Z","iopub.execute_input":"2022-06-30T07:39:26.919045Z","iopub.status.idle":"2022-06-30T07:39:28.989765Z","shell.execute_reply.started":"2022-06-30T07:39:26.918998Z","shell.execute_reply":"2022-06-30T07:39:28.988349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hy_train.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-06-30T07:41:54.586945Z","iopub.execute_input":"2022-06-30T07:41:54.587314Z","iopub.status.idle":"2022-06-30T07:41:54.597588Z","shell.execute_reply.started":"2022-06-30T07:41:54.587286Z","shell.execute_reply":"2022-06-30T07:41:54.596306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Technique 2.","metadata":{"execution":{"iopub.status.busy":"2022-06-30T07:25:39.536545Z","iopub.execute_input":"2022-06-30T07:25:39.536954Z","iopub.status.idle":"2022-06-30T07:25:39.542514Z","shell.execute_reply.started":"2022-06-30T07:25:39.536914Z","shell.execute_reply":"2022-06-30T07:25:39.540963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}