{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":5357,"databundleVersionId":38056,"sourceType":"competition"}],"dockerImageVersionId":30213,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Bosch Production Line Performance\n\n\n- 대용량 데이터인 공정 데이터를 활용하여, product의 불량 여부를 판단하는 문제입니다.\n\n\n- 불량 검출은 대표적인 class imbalance가 심한 문제입니다. class imbalance를 해결하는 sampling 방법을 사용해봅니다.\n\n\n- 공정 데이터는 대표적인 고차원 데이터입니다. 고차원 데이터를 해결하는 방법을 배워봅니다.\n\n\n- metric은 F1 score를 사용합니다.\n\n\n[Source] https://www.kaggle.com/competitions/bosch-production-line-performance","metadata":{}},{"cell_type":"markdown","source":"## 1. 라이브러리, 데이터 불러오기","metadata":{"id":"bd2f7530"}},{"cell_type":"code","source":"# 데이터분석 4종 세트\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport gc\n\n# 모델들, 성능 평가\n# (저는 일반적으로 정형데이터로 머신러닝 분석할 때는 이 2개 모델은 그냥 돌려봅니다. 특히 RF가 테스트하기 좋습니다.)\nfrom xgboost import XGBClassifier\n\n# KFold(CV), partial : optuna를 사용하기 위함\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import f1_score\nfrom functools import partial\n\n# hyper-parameter tuning을 위한 라이브러리, optuna\nimport optuna","metadata":{"id":"125fc348","execution":{"iopub.status.busy":"2024-07-24T15:47:42.091335Z","iopub.execute_input":"2024-07-24T15:47:42.091899Z","iopub.status.idle":"2024-07-24T15:47:44.636372Z","shell.execute_reply.started":"2024-07-24T15:47:42.091788Z","shell.execute_reply":"2024-07-24T15:47:44.634990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sampling = 'under' ## 'under' or 'over' or 'hybrid'","metadata":{"id":"0e4d49ce","execution":{"iopub.status.busy":"2024-07-24T15:47:44.638566Z","iopub.execute_input":"2024-07-24T15:47:44.639117Z","iopub.status.idle":"2024-07-24T15:47:44.655944Z","shell.execute_reply.started":"2024-07-24T15:47:44.639071Z","shell.execute_reply":"2024-07-24T15:47:44.654167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 데이터를 불러옵니다.\n# train_cat_part = pd.read_csv('../input/bosch-production-line-performance/train_categorical.csv.zip',\n#                             nrows=1000)\ntrain_num_part = pd.read_csv('../input/bosch-production-line-performance/train_numeric.csv.zip',\n                            nrows=1000)\ntrain_date_part = pd.read_csv('../input/bosch-production-line-performance/train_date.csv.zip',\n                             nrows=1000)\n\n#print(train_cat_part.shape, train_num_part.shape, train_date_part.shape)\nprint(train_num_part.shape, train_date_part.shape)","metadata":{"id":"3615c24a","execution":{"iopub.status.busy":"2024-07-24T15:47:44.658702Z","iopub.execute_input":"2024-07-24T15:47:44.659704Z","iopub.status.idle":"2024-07-24T15:47:45.028927Z","shell.execute_reply.started":"2024-07-24T15:47:44.659646Z","shell.execute_reply":"2024-07-24T15:47:45.027653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"categorical data는 EDA 결과 분석에 큰 도움이 없어, 사용하지 않겠습니다.","metadata":{}},{"cell_type":"code","source":"# total = train_cat_part.count() * 0\n# N = 0\n\n# for chunk in pd.read_csv('../input/bosch-production-line-performance/train_categorical.csv.zip',\n#                         chunksize=100000):  # chunksize가 클수록, 속도는 빨라지고 메모리 사용량이 늘어납니다.\n    \n#     total = total + chunk.isnull().sum()  # 전체 데이터의 column별 결측치 개수\n#     N = N + len(chunk)  # 전체 데이터의 row 수\n    \n# display(total)\n# print(N)\n# usecols = total.index[(total / N) < 0.5] # train_categorical.csv에 있는 결측치가 50% 미만인 column들.\n# usecols","metadata":{"execution":{"iopub.status.busy":"2024-07-24T15:47:45.031535Z","iopub.execute_input":"2024-07-24T15:47:45.031942Z","iopub.status.idle":"2024-07-24T15:47:45.037197Z","shell.execute_reply.started":"2024-07-24T15:47:45.031907Z","shell.execute_reply":"2024-07-24T15:47:45.035836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_cat = pd.read_csv('../input/bosch-production-line-performance/train_categorical.csv.zip',\n#                        usecols=usecols)\n# train_cat","metadata":{"execution":{"iopub.status.busy":"2024-07-24T15:47:45.038857Z","iopub.execute_input":"2024-07-24T15:47:45.039296Z","iopub.status.idle":"2024-07-24T15:47:45.050180Z","shell.execute_reply.started":"2024-07-24T15:47:45.039259Z","shell.execute_reply":"2024-07-24T15:47:45.048665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"train_date.csv에서는 파생 변수를 만들어서 사용합니다.\n\n> 1. 각 station별로 측정 시간이 모두 있거나, 없거나의 케이스만 존재한다.\n>> 데이터 중에서 station별로 하나의 column만 뽑아서 체크해도 괜찮다.\n\n> 2. 각 station별로 시간이 모두 같지는 않다. 중간에 측정시간이 바뀌는 케이스도 있다.\n>> 데이터 별로 min, max값을 계산하면 된다.\n\n- start_station : 해당 product가 처음으로 공정을 거친 station_name\n\n- end_station : 해당 product가 마지막으로 공정을 거친 station_name\n\n- time_diff : 해당 product가 처음 시작한 timestamp와 마지막으로 끝낸 timestamp의 차이 (공정에 걸린 시간)","metadata":{}},{"cell_type":"code","source":"# 1157 cols -> station 개수(S0 ~ S51)\n# 1. column들이 나와있는 series를 하나 만듭니다.\ntrain_date_part.count()\n# 2. 그 series의 index를 뽑아서 가운데 _S0_ 를 뽑습니다.\ntrain_date_part.count().reset_index()['index'].str.split('_', expand=True)[1]\n# 3. 각 station별로 하나씩만 남깁니다. + Id column 제거\ntrain_date_part.count().reset_index()['index'].str.split('_', expand=True)[1].drop_duplicates()[1:]\n# 4. 각 station별 index에 해당하는 column들을 뽑습니다.\ntemp = train_date_part.count().reset_index()['index'].str.split('_', expand=True)[1].drop_duplicates()[1:].index\ndatecols = train_date_part.columns[temp]\ndatecols","metadata":{"execution":{"iopub.status.busy":"2024-07-24T15:47:45.052074Z","iopub.execute_input":"2024-07-24T15:47:45.052974Z","iopub.status.idle":"2024-07-24T15:47:45.131719Z","shell.execute_reply.started":"2024-07-24T15:47:45.052879Z","shell.execute_reply":"2024-07-24T15:47:45.130356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"train_date_part 데이터를 가지고, 원하는 3개의 파생변수가 잘 생성되는지 확인합니다.","metadata":{}},{"cell_type":"code","source":"train_date_part[\"start_station\"] = -1  # 초기화(initialization)\ntrain_date_part[\"end_station\"] = -1\n\nfor col in train_date_part[datecols]:\n    ## TODO : start_station과 end_station을 찾아주세요\n    station_name = col.split(\"_\")[1] # S0\n    station_num = int(col.split(\"_\")[1][1:]) # 0\n    notnulls = (~train_date_part[col].isnull()) # 해당 column에서 NaN 아닌 값들의 위치 (boolean mask)\n    \n    # NaN이라면 해당 station을 지나간 것. 아니라면 해당 station에서 측정이 된 것.\n    # start_station은 해당 column에서 \"처음으로\" 수치값이 등장해야한다.(NaN이 아님) 즉, start_station이 -1이며 notnull이어야 함.\n    # end_station은 해당 column에 수치값이 등장하면 매번 update한다. 즉, notnull이기만 하면 됨.\n    train_date_part.loc[(notnulls & (train_date_part.start_station == -1)), \"start_station\"] = station_num\n    train_date_part.loc[notnulls, \"end_station\"] = station_num\n\nstart_time = train_date_part.drop(columns=[\"Id\", \"start_station\", \"end_station\"]).min(axis=1) # start_time\nend_time = train_date_part.drop(columns=[\"Id\", \"start_station\", \"end_station\"]).max(axis=1) # end_time\ntime_diff = end_time - start_time  # 공정에 걸린 시간.\ntrain_date_part['time_diff'] = time_diff\ntrain_date = train_date_part[[\"Id\", \"start_station\", \"end_station\", \"time_diff\"]]\ntrain_date","metadata":{"execution":{"iopub.status.busy":"2024-07-24T15:47:45.133871Z","iopub.execute_input":"2024-07-24T15:47:45.134502Z","iopub.status.idle":"2024-07-24T15:47:45.286055Z","shell.execute_reply.started":"2024-07-24T15:47:45.134445Z","shell.execute_reply":"2024-07-24T15:47:45.284473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## TO-DO: train_date.csv를 불러와서, start_station, end_station, time_diff 파생 변수를 생성한 뒤에,\n##        1183747 x 4 에 해당하는 train_date DataFrame을 만들어주세요.\n\n## [HINT]\n# 1. chunk (datecols)\n# 2. start, end station 만들었던거 그대로 사용\n# 3. min, max 계산해서 time_diff 구하기\nextract_cols = [\"Id\", \"start_station\", \"end_station\", \"time_diff\"]\ntrain_date = pd.DataFrame(columns=extract_cols)\n\n# file I/O\nfor chunk in pd.read_csv('../input/bosch-production-line-performance/train_date.csv.zip',\n                        usecols=[\"Id\"] + datecols.tolist(), chunksize=300000):\n\n    chunk[\"start_station\"] = -1  # 초기화(initialization)\n    chunk[\"end_station\"] = -1\n\n    for col in chunk[datecols]: # Id, start_station, end_station 제외한것과 같음.\n        ## TODO : start_station과 end_station을 찾아주세요\n        station_num = int(col.split(\"_\")[1][1:])\n        notnulls = (~chunk[col].isnull()) # 해당 column에서 NaN 아닌 값들의 위치 (boolean mask)\n\n        chunk.loc[(notnulls & (chunk.start_station == -1)), \"start_station\"] = station_num\n        chunk.loc[notnulls, \"end_station\"] = station_num\n        \n        \n    start_time = chunk[datecols].min(axis=1)\n    end_time = chunk[datecols].max(axis=1)\n    time_diff = end_time - start_time\n    chunk['time_diff'] = time_diff\n    chunk = chunk[extract_cols]\n    train_date = pd.concat([train_date, chunk])\n    \ntrain_date","metadata":{"execution":{"iopub.status.busy":"2024-07-24T15:47:45.288080Z","iopub.execute_input":"2024-07-24T15:47:45.288586Z","iopub.status.idle":"2024-07-24T15:48:32.487227Z","shell.execute_reply.started":"2024-07-24T15:47:45.288538Z","shell.execute_reply":"2024-07-24T15:48:32.485705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_numeric.csv에서 결측치가 50% 이하인 데이터.\ntotal = train_num_part.count() * 0\nN = 0\n\nfor chunk in pd.read_csv('../input/bosch-production-line-performance/train_numeric.csv.zip',\n                        chunksize=100000):  # chunksize가 클수록, 속도는 빨라지고 메모리 사용량이 늘어납니다.\n    \n    total = total + chunk.isnull().sum()  # 전체 데이터의 column별 결측치 개수\n    N = N + len(chunk)  # 전체 데이터의 row 수\n    \ndisplay(total)\nprint(N)\nusecols = total.index[(total / N) < 0.5] # train_numeric.csv에 있는 결측치가 50% 미만인 column들.\nusecols","metadata":{"execution":{"iopub.status.busy":"2024-07-24T15:48:32.488757Z","iopub.execute_input":"2024-07-24T15:48:32.489228Z","iopub.status.idle":"2024-07-24T15:50:15.977497Z","shell.execute_reply.started":"2024-07-24T15:48:32.489182Z","shell.execute_reply":"2024-07-24T15:50:15.976302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##\ntrain_num = pd.read_csv('../input/bosch-production-line-performance/train_numeric.csv.zip',\n                       usecols=usecols)\ntrain = pd.merge(train_num, train_date, on='Id')\nprint(train.shape)\n\n# 메모리 확보\ndel train_num\ndel train_date\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-24T15:50:15.981648Z","iopub.execute_input":"2024-07-24T15:50:15.982067Z","iopub.status.idle":"2024-07-24T15:51:08.912654Z","shell.execute_reply.started":"2024-07-24T15:50:15.982022Z","shell.execute_reply":"2024-07-24T15:51:08.911364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 남은 결측치를 측정되지 않은 값으로 처리하여 0으로 채움.\ntrain = train.fillna(0)\ntrain.info(memory_usage='deep')","metadata":{"execution":{"iopub.status.busy":"2024-07-24T15:51:08.914294Z","iopub.execute_input":"2024-07-24T15:51:08.915699Z","iopub.status.idle":"2024-07-24T15:51:12.163672Z","shell.execute_reply.started":"2024-07-24T15:51:08.915647Z","shell.execute_reply":"2024-07-24T15:51:12.162186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. EDA","metadata":{"id":"c9c9acb8"}},{"cell_type":"markdown","source":"- 데이터에서 찾아야 하는 기초적인 내용들을 확인합니다.\n\n\n- class imbalance, target distribution, outlier, correlation을 확인합니다.","metadata":{"id":"6fdf620b"}},{"cell_type":"code","source":"## On your Own\ndisplay(train.Response.value_counts()) # 0.995 : 0.005 = 995 : 5 = 199 : 1\nsns.countplot(data=train, x=\"Response\")","metadata":{"id":"adb06474","execution":{"iopub.status.busy":"2024-07-24T15:51:12.165283Z","iopub.execute_input":"2024-07-24T15:51:12.165686Z","iopub.status.idle":"2024-07-24T15:51:12.434712Z","shell.execute_reply.started":"2024-07-24T15:51:12.165645Z","shell.execute_reply":"2024-07-24T15:51:12.433316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"이런 식으로 여러가지 그래프를 그려가며, 데이터에 대한 인사이트를 얻습니다!","metadata":{"id":"f2c602bd"}},{"cell_type":"markdown","source":"### 3. 전처리\n\nClass Imbalance가 존재하는 경우에는 data sampling 기법을 이용하여 문제를 해결합니다.\n- majority class : 다수가 속해있는 class\n\n- minority class : 소수가 속해있는 class\n\n\n\n**1) Under sampling : majority class의 수를 minority class로 맞춰주기 위해서 random sampling을 수행하는 방법.**\n\ne.g. 1176868 : 6879 -> 6879 : 6879\n\n\n**2) Over sampling : minority class에서 데이터를 생성하여 majority class의 수를 맞춰주는 방법.**\n\ne.g. 1176868 : 6879 -> 1176868 : 1176868\n\n\n**3) Hybrid approach : under sampling & over sampling을 써서 적당한 수로 맞춰주는 방법.**\n\ne.g. 1176868 : 6879 -> 20000 : 20000","metadata":{"id":"9dbb8802"}},{"cell_type":"code","source":"# 모든 데이터가 NaN이라서, time_diff가 NaN이고 train_date에 아무 정보가 없는 데이터들을 제거합니다.\nX_train = train.dropna() # 1183747\nX_train","metadata":{"execution":{"iopub.status.busy":"2024-07-24T15:51:12.436545Z","iopub.execute_input":"2024-07-24T15:51:12.437260Z","iopub.status.idle":"2024-07-24T15:51:14.121243Z","shell.execute_reply.started":"2024-07-24T15:51:12.437213Z","shell.execute_reply":"2024-07-24T15:51:14.119879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 1) Undersampling\nif sampling == 'under':\n    N = 50000\n    normal = X_train[X_train.Response == 0]\n    abnormal = X_train[X_train.Response == 1]\n    # do undersampling\n    normal = normal.sample(n=N, random_state=42)\n    print(normal.shape, abnormal.shape)\n    \n    temp = pd.concat([normal, abnormal])\n    X_s = temp.drop(columns=[\"Id\", \"Response\"])\n    y_s = temp.Response","metadata":{"execution":{"iopub.status.busy":"2024-07-24T15:51:14.122882Z","iopub.execute_input":"2024-07-24T15:51:14.123258Z","iopub.status.idle":"2024-07-24T15:51:15.311720Z","shell.execute_reply.started":"2024-07-24T15:51:14.123225Z","shell.execute_reply":"2024-07-24T15:51:15.310391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if sampling == 'over':\n    # 2) Oversampling - SMOTE (only numeric features)\n    from imblearn.over_sampling import SMOTE\n\n    X = X_train.drop(columns=[\"Id\", \"Response\"])\n    y = X_train.Response\n\n    X_s, y_s = SMOTE(random_state=42).fit_resample(X, y)\n    print(X_s.shape, y_s.shape)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T15:51:15.313238Z","iopub.execute_input":"2024-07-24T15:51:15.313669Z","iopub.status.idle":"2024-07-24T15:51:15.320928Z","shell.execute_reply.started":"2024-07-24T15:51:15.313602Z","shell.execute_reply":"2024-07-24T15:51:15.319658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if sampling == 'hybrid':\n    # 3) Hybrid approach\n    N_sample = 20000\n    normal = X_train[X_train.Response == 0].sample(n=N_sample, random_state=42) # 10000개 샘플\n    abnormal = X_train[X_train.Response == 1]\n    temp = pd.concat([normal, abnormal]) # 10000 + 6877\n\n    from imblearn.over_sampling import SMOTE\n\n    X = temp.drop(columns=[\"Id\", \"Response\"])\n    y = temp.Response\n\n    X_s, y_s = SMOTE(random_state=42).fit_resample(X, y)\n    print(X_s.shape, y_s.shape) # (20000, 43), (20000, )","metadata":{"execution":{"iopub.status.busy":"2024-07-24T15:51:15.322393Z","iopub.execute_input":"2024-07-24T15:51:15.322802Z","iopub.status.idle":"2024-07-24T15:51:15.335744Z","shell.execute_reply.started":"2024-07-24T15:51:15.322767Z","shell.execute_reply":"2024-07-24T15:51:15.334320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# after sampling, convert float to int.\nX_s['start_station'] = X_s.start_station.astype(np.int8) # 64 -> 8 (reduce x8)\nX_s['end_station'] = X_s.end_station.astype(np.int8)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T15:51:15.337542Z","iopub.execute_input":"2024-07-24T15:51:15.338078Z","iopub.status.idle":"2024-07-24T15:51:15.355989Z","shell.execute_reply.started":"2024-07-24T15:51:15.338017Z","shell.execute_reply":"2024-07-24T15:51:15.354250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4. 학습 데이터 분할","metadata":{"id":"f497a2d8"}},{"cell_type":"code","source":"# 첫번째 테스트용으로 사용하고, 실제 학습시에는 K-Fold CV를 사용합니다.\n# train : test = 8 : 2\nfrom sklearn.model_selection import train_test_split\n\nX = X_s\ny = y_s\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)\nprint(X_train.shape, X_val.shape, y_train.shape, y_val.shape)","metadata":{"id":"47306aaf","execution":{"iopub.status.busy":"2024-07-24T15:51:15.357693Z","iopub.execute_input":"2024-07-24T15:51:15.358445Z","iopub.status.idle":"2024-07-24T15:51:15.450742Z","shell.execute_reply.started":"2024-07-24T15:51:15.358393Z","shell.execute_reply":"2024-07-24T15:51:15.449185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 5. 학습 및 평가","metadata":{"id":"58056e51"}},{"cell_type":"code","source":"clf = XGBClassifier()\nclf.fit(X_train, y_train)","metadata":{"id":"39fd2515","execution":{"iopub.status.busy":"2024-07-24T15:51:15.452677Z","iopub.execute_input":"2024-07-24T15:51:15.453142Z","iopub.status.idle":"2024-07-24T15:51:31.946604Z","shell.execute_reply.started":"2024-07-24T15:51:15.453095Z","shell.execute_reply":"2024-07-24T15:51:31.945316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Prediction\")\npred_train = clf.predict(X_train)\npred_test = clf.predict(X_val)\n\ntrain_score = f1_score(y_train, pred_train)\ntest_score = f1_score(y_val, pred_test)\n\nprint(\"Train Score : %.4f\" % train_score)\nprint(\"Test Score : %.4f\" % test_score)","metadata":{"id":"a6b39be5","execution":{"iopub.status.busy":"2024-07-24T15:51:31.948269Z","iopub.execute_input":"2024-07-24T15:51:31.948776Z","iopub.status.idle":"2024-07-24T15:51:32.139330Z","shell.execute_reply.started":"2024-07-24T15:51:31.948729Z","shell.execute_reply":"2024-07-24T15:51:32.138033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 6. Hyper-parameter Tuning","metadata":{"id":"bc755b17"}},{"cell_type":"markdown","source":"> optuna를 사용해봅시다 !","metadata":{"id":"97e5302d"}},{"cell_type":"code","source":"def optimizer(trial, X, y, K):\n    # 조절할 hyper-parameter 조합을 적어줍니다.\n    n_estimators = trial.suggest_int('n_estimators', 200, 500)\n    max_depth = trial.suggest_int('max_depth', 5, 8)\n    learning_rate = trial.suggest_float('learning_rate', 0.01, 0.3)\n    \n    model = XGBClassifier(n_estimators=n_estimators,\n                          max_depth=max_depth,\n                          learning_rate=learning_rate,\n                          colsample_bynode=0.5,\n#                           gpu_id=0,\n                          random_state=42)\n    \n    # K-Fold Cross validation을 구현합니다.\n    from sklearn.model_selection import StratifiedKFold\n    folds = StratifiedKFold(n_splits=K, shuffle=True, random_state=42)\n    scores = []\n    \n    for train_idx, val_idx in folds.split(X, y):\n        X_train = X.iloc[train_idx, :]\n        y_train = y.iloc[train_idx]\n        \n        X_val = X.iloc[val_idx, :]\n        y_val = y.iloc[val_idx]\n        \n        model.fit(X_train, y_train)\n        preds = model.predict(X_val)\n        score = f1_score(y_val, preds)\n        scores.append(score)\n    \n    del model\n    gc.collect()\n    # K-Fold의 평균 loss값을 돌려줍니다.\n    return np.mean(scores)","metadata":{"id":"34ce4986","execution":{"iopub.status.busy":"2024-07-24T16:19:14.399534Z","iopub.execute_input":"2024-07-24T16:19:14.400036Z","iopub.status.idle":"2024-07-24T16:19:14.411971Z","shell.execute_reply.started":"2024-07-24T16:19:14.399989Z","shell.execute_reply":"2024-07-24T16:19:14.410424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"K = 5 # Kfold 수\nopt_func = partial(optimizer, X=X, y=y, K=K)\n\nstudy = optuna.create_study(direction=\"maximize\") # 최소/최대 어느 방향의 최적값을 구할 건지.\nstudy.optimize(opt_func, n_trials=30)","metadata":{"id":"7150b210","execution":{"iopub.status.busy":"2024-07-24T16:19:15.204257Z","iopub.execute_input":"2024-07-24T16:19:15.205419Z","iopub.status.idle":"2024-07-24T17:44:03.415386Z","shell.execute_reply.started":"2024-07-24T16:19:15.205370Z","shell.execute_reply":"2024-07-24T17:44:03.411683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# optuna가 시도했던 모든 실험 관련 데이터\nstudy.trials_dataframe()","metadata":{"id":"72d0a118","execution":{"iopub.status.busy":"2024-07-24T17:44:07.790145Z","iopub.execute_input":"2024-07-24T17:44:07.790579Z","iopub.status.idle":"2024-07-24T17:44:07.821776Z","shell.execute_reply.started":"2024-07-24T17:44:07.790542Z","shell.execute_reply":"2024-07-24T17:44:07.820348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best Score: %.4f\" % study.best_value) # best score 출력\nprint(\"Best params: \", study.best_trial.params) # best score일 때의 하이퍼파라미터들","metadata":{"id":"a805da05","execution":{"iopub.status.busy":"2024-07-24T17:44:08.036803Z","iopub.execute_input":"2024-07-24T17:44:08.037225Z","iopub.status.idle":"2024-07-24T17:44:08.044654Z","shell.execute_reply.started":"2024-07-24T17:44:08.037184Z","shell.execute_reply":"2024-07-24T17:44:08.042929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 실험 기록 시각화\noptuna.visualization.plot_optimization_history(study)","metadata":{"id":"051ae1eb","execution":{"iopub.status.busy":"2024-07-24T17:44:08.269461Z","iopub.execute_input":"2024-07-24T17:44:08.269897Z","iopub.status.idle":"2024-07-24T17:44:08.436633Z","shell.execute_reply.started":"2024-07-24T17:44:08.269860Z","shell.execute_reply":"2024-07-24T17:44:08.435441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# hyper-parameter들의 중요도\noptuna.visualization.plot_param_importances(study)","metadata":{"id":"efbf8f65","execution":{"iopub.status.busy":"2024-07-24T17:44:11.578516Z","iopub.execute_input":"2024-07-24T17:44:11.580481Z","iopub.status.idle":"2024-07-24T17:44:12.036635Z","shell.execute_reply.started":"2024-07-24T17:44:11.580413Z","shell.execute_reply":"2024-07-24T17:44:12.035192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 7. 테스트 및 제출 파일 생성","metadata":{"id":"24b360ec"}},{"cell_type":"code","source":"# release memory\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-24T17:44:13.100313Z","iopub.execute_input":"2024-07-24T17:44:13.100792Z","iopub.status.idle":"2024-07-24T17:44:13.247559Z","shell.execute_reply.started":"2024-07-24T17:44:13.100753Z","shell.execute_reply":"2024-07-24T17:44:13.246075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_num_part = pd.read_csv('../input/bosch-production-line-performance/test_numeric.csv.zip',\n                           nrows=1000)\ntest_date_part = pd.read_csv('../input/bosch-production-line-performance/test_date.csv.zip',\n                            nrows=1000)","metadata":{"execution":{"iopub.status.busy":"2024-07-24T17:44:13.354274Z","iopub.execute_input":"2024-07-24T17:44:13.354730Z","iopub.status.idle":"2024-07-24T17:44:13.681432Z","shell.execute_reply.started":"2024-07-24T17:44:13.354693Z","shell.execute_reply":"2024-07-24T17:44:13.680008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 전처리 과정은 train data와 동일하게 진행합니다.\n\n1. test_numeric : 위에서 사용한 usecols에 해당하는 column만 read_csv를 통해서 가져옵니다.\n\n2. test_date : start_station, end_station, time_diff의 파생변수를 생성한 방법과 동일한 처리를 해줍니다.\n\n3. X_test : 위의 2개의 DataFrame을 합칩니다.","metadata":{}},{"cell_type":"code","source":"## TO-DO: train_date.csv를 불러와서, start_station, end_station, time_diff 파생 변수를 생성한 뒤에,\n##        1183747 x 4 에 해당하는 train_date DataFrame을 만들어주세요.\n\n## [HINT]\n# 1. chunk (usecols)\n# 2. start, end station 만들었던거 그대로 사용\n# 3. min, max 계산해서 time_diff 구하기\nextract_cols = [\"Id\", \"start_station\", \"end_station\", \"time_diff\"]\ntest_date = pd.DataFrame(columns=extract_cols)\n\nfor chunk in pd.read_csv('../input/bosch-production-line-performance/test_date.csv.zip',\n                        usecols=[\"Id\"] + datecols.tolist(), chunksize=100000):\n\n    chunk[\"start_station\"] = -1  # 초기화(initialization)\n    chunk[\"end_station\"] = -1\n\n    for col in chunk[datecols]: # Id, start_station, end_station 제외한것과 같음.\n        ## TODO : start_station과 end_station을 찾아주세요\n        station_num = int(col.split(\"_\")[1][1:])\n        notnulls = (~chunk[col].isnull()) # 해당 column에서 NaN 아닌 값들의 위치 (boolean mask)\n\n        chunk.loc[(notnulls & (chunk.start_station == -1)), \"start_station\"] = station_num\n        chunk.loc[notnulls, \"end_station\"] = station_num\n        \n        \n    start_time = chunk[datecols].min(axis=1)\n    end_time = chunk[datecols].max(axis=1)\n    time_diff = end_time - start_time\n    chunk['time_diff'] = time_diff\n    chunk = chunk[extract_cols]\n    test_date = pd.concat([test_date, chunk])\n    \ntest_date","metadata":{"execution":{"iopub.status.busy":"2024-07-24T17:44:13.902230Z","iopub.execute_input":"2024-07-24T17:44:13.902709Z","iopub.status.idle":"2024-07-24T17:44:59.646114Z","shell.execute_reply.started":"2024-07-24T17:44:13.902669Z","shell.execute_reply":"2024-07-24T17:44:59.644862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"usecols = usecols.drop('Response')\ntest_num = pd.read_csv('../input/bosch-production-line-performance/test_numeric.csv.zip', usecols=usecols)\nX_test = pd.merge(test_num, test_date, on='Id')\nX_test = X_test.fillna(0).drop(columns=['Id'])\nX_test.info(memory_usage='deep')","metadata":{"execution":{"iopub.status.busy":"2024-07-24T17:44:59.648095Z","iopub.execute_input":"2024-07-24T17:44:59.648488Z","iopub.status.idle":"2024-07-24T17:45:58.783259Z","shell.execute_reply.started":"2024-07-24T17:44:59.648452Z","shell.execute_reply":"2024-07-24T17:45:58.781919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_params = study.best_trial.params\n\nmodel = XGBClassifier(**best_params, colsample_bynode=0.5, random_state=42)#, gpu_id=0) # 찾은 best hyper-parameter.\n\nmodel.fit(X, y) # 전체 데이터에 대해서 학습! (model finalization)\npreds = model.predict(X_test)\npreds","metadata":{"id":"a0daf54e","execution":{"iopub.status.busy":"2024-07-24T17:45:58.784786Z","iopub.execute_input":"2024-07-24T17:45:58.785162Z","iopub.status.idle":"2024-07-24T17:47:30.463102Z","shell.execute_reply.started":"2024-07-24T17:45:58.785129Z","shell.execute_reply":"2024-07-24T17:47:30.461783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1_score(y, model.predict(X))","metadata":{"execution":{"iopub.status.busy":"2024-07-24T17:47:30.466049Z","iopub.execute_input":"2024-07-24T17:47:30.466497Z","iopub.status.idle":"2024-07-24T17:47:31.084074Z","shell.execute_reply.started":"2024-07-24T17:47:30.466456Z","shell.execute_reply":"2024-07-24T17:47:31.082586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n\nConfusionMatrixDisplay.from_estimator(model, X, y, values_format='d')","metadata":{"execution":{"iopub.status.busy":"2024-07-24T17:47:31.085917Z","iopub.execute_input":"2024-07-24T17:47:31.086373Z","iopub.status.idle":"2024-07-24T17:47:31.911037Z","shell.execute_reply.started":"2024-07-24T17:47:31.086337Z","shell.execute_reply":"2024-07-24T17:47:31.909702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 제출 파일 불러와서, 예측값 채우기\nsubmission = pd.read_csv(\"../input/bosch-production-line-performance/sample_submission.csv.zip\")\nsubmission['Response'] = preds","metadata":{"id":"8ff2070c","execution":{"iopub.status.busy":"2024-07-24T17:47:31.912454Z","iopub.execute_input":"2024-07-24T17:47:31.912833Z","iopub.status.idle":"2024-07-24T17:47:32.136876Z","shell.execute_reply.started":"2024-07-24T17:47:31.912798Z","shell.execute_reply":"2024-07-24T17:47:32.135681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","metadata":{"id":"55a2c13f","execution":{"iopub.status.busy":"2024-07-24T17:47:32.138575Z","iopub.execute_input":"2024-07-24T17:47:32.138974Z","iopub.status.idle":"2024-07-24T17:47:33.501099Z","shell.execute_reply.started":"2024-07-24T17:47:32.138938Z","shell.execute_reply":"2024-07-24T17:47:33.499732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}