{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Set Common Config","metadata":{"id":"PpDQRHvVasIT"}},{"cell_type":"code","source":"IS_INFERENCE = False\nUSE_MADE_FEATURES = False\n\nIS_KAGGLE = True\nSEED = 42\nN_FOLD = 5\nPROGRAM_VERSION = \"900_CodeBERT\"\nGRADIENT_ACCUMULATION_STEPS = 4  # nbme 2\nEPOCHS = 5\nNUM_WORKERS = 8\nMD_MAX_LEN = 64\nTOTAL_MAX_LEN = 512\nBATCH_SIZE = 8\nif IS_INFERENCE: BATCH_SIZE = 32\n\n\nMODEL_NAME = 'microsoft/codebert-base'\n# MODEL_NAME = 'microsoft/graphcodebert-base'\n\n\nDATA_PATH = '/content/drive/MyDrive/kaggle/ai4code/data/'\nSUBMISSION_PATH = '/content/drive/MyDrive/kaggle/ai4code/sub/'\nMODEL_PATH = '/content/drive/MyDrive/kaggle/ai4code/model/'\nCLEANED_PATH = '/content/drive/MyDrive/kaggle/ai4code/clean/'\nCONFIG_PATH = MODEL_PATH + 'config.pth'\nTOKENIZER = None\nif IS_KAGGLE:\n    DATA_PATH = '../input/AI4Code/'\n    SUBMISSION_PATH = './'\n    MODEL_PATH = './ai4code-model-codebert-base/'  # 기본 모델 생성 디렉토리\n    CLEANED_PATH = './ai4code-clean-codebert-base/'\n    CONFIG_PATH = MODEL_PATH + 'config.pth'\n    if IS_INFERENCE:  # inference만 하는 경우\n        MODEL_PATH = '../input/nbme-model-codebert-base/'\n        CLEANED_PATH = '../input/nbme-clean-codebert-base/'\n        CONFIG_PATH = MODEL_PATH + 'config.pth'\n        NUM_WORKERS = 0\n","metadata":{"id":"lexJt5IVBB4A","execution":{"iopub.status.busy":"2022-06-11T00:29:48.960643Z","iopub.execute_input":"2022-06-11T00:29:48.961185Z","iopub.status.idle":"2022-06-11T00:29:48.972330Z","shell.execute_reply.started":"2022-06-11T00:29:48.961142Z","shell.execute_reply":"2022-06-11T00:29:48.971120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# colab에서 실행하는 경우 google drive에 연결 및 colab 서버에 drive mount\nif not IS_KAGGLE:\n    from google.colab import drive\n    drive.mount('/content/drive')","metadata":{"id":"RTkOq6TMBB4B","outputId":"ef493c94-ead3-4d49-9cc1-dc09b79b7296","execution":{"iopub.status.busy":"2022-06-11T00:29:55.509651Z","iopub.execute_input":"2022-06-11T00:29:55.510166Z","iopub.status.idle":"2022-06-11T00:29:55.516401Z","shell.execute_reply.started":"2022-06-11T00:29:55.510127Z","shell.execute_reply":"2022-06-11T00:29:55.515185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import Basic Modules","metadata":{"id":"OGkhvGWkksSH"}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n%matplotlib inline\nimport matplotlib.pyplot as plt\nimport re\nimport platform\nfrom tqdm import tqdm\ntqdm.pandas()\nimport os\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\nimport random\nimport collections\nimport json\nimport gc\nfrom pathlib import Path\nfrom scipy import sparse\nfrom bisect import bisect\nimport sys\n\nimport warnings; warnings.filterwarnings(\"ignore\")\nfrom scipy.stats import gmean\n\nimport tensorflow as tf\nfrom keras import layers\nfrom keras.layers import Embedding, Dense\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom keras.regularizers import l2\nfrom keras.models import Model\n\nfrom transformers import TFAutoModel, AutoTokenizer\nfrom transformers import AutoModel, AutoTokenizer, AutoConfig, AdamW, get_linear_schedule_with_warmup\n\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torch\n\n\nfrom sklearn.model_selection import GroupShuffleSplit\nfrom sklearn.model_selection import GroupKFold, KFold, StratifiedKFold","metadata":{"id":"94lEG53S4xm9","execution":{"iopub.status.busy":"2022-06-11T00:29:56.656552Z","iopub.execute_input":"2022-06-11T00:29:56.657045Z","iopub.status.idle":"2022-06-11T00:30:05.711462Z","shell.execute_reply.started":"2022-06-11T00:29:56.657009Z","shell.execute_reply":"2022-06-11T00:30:05.710619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint('device : ', device)","metadata":{"id":"U-focurQN_Wr","outputId":"251ed9a1-43f9-4a86-c3cd-7d5759e7fe1f","execution":{"iopub.status.busy":"2022-06-11T00:30:05.712988Z","iopub.execute_input":"2022-06-11T00:30:05.714038Z","iopub.status.idle":"2022-06-11T00:30:05.719312Z","shell.execute_reply.started":"2022-06-11T00:30:05.714002Z","shell.execute_reply":"2022-06-11T00:30:05.718445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define functions","metadata":{"id":"YlkzsDxCmpHD"}},{"cell_type":"markdown","source":"- 재현성 구성함수","metadata":{"id":"Ipgetq6-5P7A"}},{"cell_type":"code","source":"def reset_seeds(seed):\n    np.random.seed(seed)\n    random.seed(seed)\n    tf.compat.v1.set_random_seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)    \n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    \n    os.environ['CUDA_VISIBLE_DEVICES'] = ''  # for GPU\n    print(\"RANDOM SEEDS RESET {}\".format(seed))  # optional\n\nreset_seeds(SEED)","metadata":{"id":"0Fh7Va0YBB4C","outputId":"0fb4da30-5c75-4d1d-8457-6b612876b6fc","execution":{"iopub.status.busy":"2022-06-11T00:30:08.657365Z","iopub.execute_input":"2022-06-11T00:30:08.658492Z","iopub.status.idle":"2022-06-11T00:30:08.670466Z","shell.execute_reply.started":"2022-06-11T00:30:08.658450Z","shell.execute_reply":"2022-06-11T00:30:08.669370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 데이타 읽기","metadata":{"id":"_O4PPsBq5P7B"}},{"cell_type":"code","source":"def read_notebook(path):\n    return (\n        pd.read_json(\n            path,\n            dtype={'cell_type': 'category', 'source': 'str'})\n            .assign(id=path.stem)\n            .rename_axis('cell_id')\n    )","metadata":{"id":"Kk0rUBFhGzGO","execution":{"iopub.status.busy":"2022-06-11T00:30:10.706214Z","iopub.execute_input":"2022-06-11T00:30:10.706644Z","iopub.status.idle":"2022-06-11T00:30:10.714191Z","shell.execute_reply.started":"2022-06-11T00:30:10.706604Z","shell.execute_reply":"2022-06-11T00:30:10.712881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 순위 적용","metadata":{"id":"kHMQhd_o5P7C"}},{"cell_type":"code","source":"def get_ranks(base, derived):\n    return [base.index(d) for d in derived]","metadata":{"id":"nwIOXqzs5P7C","execution":{"iopub.status.busy":"2022-06-11T00:30:11.027916Z","iopub.execute_input":"2022-06-11T00:30:11.028747Z","iopub.status.idle":"2022-06-11T00:30:11.034321Z","shell.execute_reply.started":"2022-06-11T00:30:11.028697Z","shell.execute_reply":"2022-06-11T00:30:11.033411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 데이타 분할","metadata":{"id":"NPKY9ht05P7C"}},{"cell_type":"code","source":"def cv_split(data, n_fold):\n    if data is None:\n        return data\n    fold = GroupKFold(n_splits=n_fold)\n    groups = data['ancestor_id'].values\n    for n, (train_index, val_index) in enumerate(fold.split(data, data['cell_type'], groups)):\n        data.loc[val_index, f'model_fold_num'] = int(n)\n    data[f'model_fold_num'] = data[f'model_fold_num'].astype(int)\n    print(data.groupby(f'model_fold_num').size())\n\n    return data","metadata":{"id":"UtmHGXgd5P7C","execution":{"iopub.status.busy":"2022-06-11T00:30:11.375396Z","iopub.execute_input":"2022-06-11T00:30:11.376202Z","iopub.status.idle":"2022-06-11T00:30:11.384197Z","shell.execute_reply.started":"2022-06-11T00:30:11.376151Z","shell.execute_reply":"2022-06-11T00:30:11.383501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 모델 입력데이타 전처리","metadata":{"id":"HpD4QPNg5P7D"}},{"cell_type":"code","source":"def clean_code(cell):\n    return str(cell).replace(\"\\\\n\", \"\\n\")","metadata":{"id":"9yGr-QA95P7D","execution":{"iopub.status.busy":"2022-06-11T00:30:11.741640Z","iopub.execute_input":"2022-06-11T00:30:11.742206Z","iopub.status.idle":"2022-06-11T00:30:11.747592Z","shell.execute_reply.started":"2022-06-11T00:30:11.742161Z","shell.execute_reply":"2022-06-11T00:30:11.746510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 모델 입력데이타 Sampling","metadata":{"id":"Wka_p7uV5P7D"}},{"cell_type":"code","source":"def sample_cells(cells, n):\n    cells = [clean_code(cell) for cell in cells]\n    if n >= len(cells):\n        return [cell[:200] for cell in cells]\n    else:\n        results = []\n        step = len(cells) / n\n        idx = 0\n        while int(np.round(idx)) < len(cells):\n            results.append(cells[int(np.round(idx))])\n            idx += step\n        assert cells[0] in results\n        if cells[-1] not in results:\n            results[-1] = cells[-1]\n        return results","metadata":{"id":"DM_rEaNy5P7D","execution":{"iopub.status.busy":"2022-06-11T00:30:12.082803Z","iopub.execute_input":"2022-06-11T00:30:12.083933Z","iopub.status.idle":"2022-06-11T00:30:12.092438Z","shell.execute_reply.started":"2022-06-11T00:30:12.083892Z","shell.execute_reply":"2022-06-11T00:30:12.091369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 모델 입력데이타 Feature Engineering","metadata":{"id":"Pf5PaYbs5P7D"}},{"cell_type":"code","source":"def get_features(df):\n    features = dict()\n    df = df.sort_values(\"rank\").reset_index(drop=True)\n    for idx, sub_df in tqdm(df.groupby(\"id\")):\n        features[idx] = dict()\n        total_md = sub_df[sub_df.cell_type == \"markdown\"].shape[0]\n        code_sub_df = sub_df[sub_df.cell_type == \"code\"]\n        total_code = code_sub_df.shape[0]\n        codes = sample_cells(code_sub_df.source.values, 20)\n        features[idx][\"total_code\"] = total_code\n        features[idx][\"total_md\"] = total_md\n        features[idx][\"codes\"] = codes\n    return features","metadata":{"id":"IttYIKkg5P7E","execution":{"iopub.status.busy":"2022-06-11T00:30:12.456868Z","iopub.execute_input":"2022-06-11T00:30:12.457835Z","iopub.status.idle":"2022-06-11T00:30:12.465848Z","shell.execute_reply.started":"2022-06-11T00:30:12.457795Z","shell.execute_reply":"2022-06-11T00:30:12.464755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Metric 함수","metadata":{"id":"znhgL1cv5P7E"}},{"cell_type":"code","source":"def count_inversions(a):\n    inversions = 0\n    sorted_so_far = []\n    for i, u in enumerate(a):\n        j = bisect(sorted_so_far, u)\n        inversions += i - j\n        sorted_so_far.insert(j, u)\n    return inversions\n\n\ndef kendall_tau(ground_truth, predictions):\n    total_inversions = 0\n    total_2max = 0  # twice the maximum possible inversions across all instances\n    for gt, pred in zip(ground_truth, predictions):\n        ranks = [gt.index(x) for x in pred]  # rank predicted order in terms of ground truth\n        total_inversions += count_inversions(ranks)\n        n = len(gt)\n        total_2max += n * (n - 1)\n    return 1 - 4 * total_inversions / total_2max","metadata":{"id":"imazZtnZ5P7E","execution":{"iopub.status.busy":"2022-06-11T00:30:13.102030Z","iopub.execute_input":"2022-06-11T00:30:13.103080Z","iopub.status.idle":"2022-06-11T00:30:13.111000Z","shell.execute_reply.started":"2022-06-11T00:30:13.103034Z","shell.execute_reply":"2022-06-11T00:30:13.109955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Read Data From Dataloader ","metadata":{"id":"jr-CD3Al5P7E"}},{"cell_type":"code","source":"def read_data(data):\n    return tuple(d.cuda() for d in data[:-1]), data[-1].cuda()","metadata":{"id":"KCKT9WwF5P7E","execution":{"iopub.status.busy":"2022-06-11T00:30:13.725126Z","iopub.execute_input":"2022-06-11T00:30:13.725980Z","iopub.status.idle":"2022-06-11T00:30:13.732034Z","shell.execute_reply.started":"2022-06-11T00:30:13.725940Z","shell.execute_reply":"2022-06-11T00:30:13.730591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 사전학습된 모델 파일 확인","metadata":{"id":"gbVx6oiv7aDf"}},{"cell_type":"code","source":"# 인수(idx)로 받은 모델번호의 모델파일명을 불러오는 함수\ndef get_model_filename(idx):\n    model_list_path = f'{MODEL_PATH}{PROGRAM_VERSION}'\n    file_list = os.listdir(model_list_path)\n    file_list = [file_name for file_name in file_list if PROGRAM_VERSION in file_name]\n    model_filename = [file_name for file_name in file_list if 'WEIGHTS_' + str(idx) in file_name]\n    if len(model_filename) == 0:\n        return 'not found'\n    return model_filename[0]\n# get_model_filename(0)","metadata":{"id":"nV9bJJ4-7ZoH","execution":{"iopub.status.busy":"2022-06-11T00:30:14.297794Z","iopub.execute_input":"2022-06-11T00:30:14.298858Z","iopub.status.idle":"2022-06-11T00:30:14.306650Z","shell.execute_reply.started":"2022-06-11T00:30:14.298817Z","shell.execute_reply":"2022-06-11T00:30:14.305455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 디렉토리 생성 함","metadata":{"id":"PIis9S1W77Ev"}},{"cell_type":"code","source":"# 디렉토리 생성 함수 (없는 경우만 신규로 생성)\ndef createFolder(directory):\n    try:\n        if not os.path.exists(directory):\n            os.makedirs(directory)\n    except OSError:\n        print ('Error: Creating directory. ' +  directory)\n\n\ncreateFolder(f'{MODEL_PATH}{PROGRAM_VERSION}')\ncreateFolder(f'{CLEANED_PATH}')","metadata":{"id":"1Gj90cdz79SK","execution":{"iopub.status.busy":"2022-06-11T01:18:55.494118Z","iopub.execute_input":"2022-06-11T01:18:55.494917Z","iopub.status.idle":"2022-06-11T01:18:55.502713Z","shell.execute_reply.started":"2022-06-11T01:18:55.494877Z","shell.execute_reply":"2022-06-11T01:18:55.501755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 데이타 전처리","metadata":{"id":"HeWaJlHkGAv7"}},{"cell_type":"code","source":"if not USE_MADE_FEATURES:\n    paths_train = list((Path(f'{DATA_PATH}train')).glob('*.json'))\n    notebooks_train = [\n        read_notebook(path) for path in tqdm(paths_train, desc='Train NBs')\n    ]","metadata":{"id":"ON-ZZYm5sX0p","outputId":"b837a42a-bf2a-489d-f10d-74a51df216e9","execution":{"iopub.status.busy":"2022-06-11T00:30:17.506149Z","iopub.execute_input":"2022-06-11T00:30:17.506515Z","iopub.status.idle":"2022-06-11T00:59:15.711704Z","shell.execute_reply.started":"2022-06-11T00:30:17.506487Z","shell.execute_reply":"2022-06-11T00:59:15.710673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not USE_MADE_FEATURES:\n    df = (\n        pd.concat(notebooks_train)\n            .set_index('id', append=True)\n            .swaplevel()\n            .sort_index(level='id', sort_remaining=False)\n    )","metadata":{"id":"q6ciThNP5J1s","execution":{"iopub.status.busy":"2022-06-11T01:08:51.367842Z","iopub.execute_input":"2022-06-11T01:08:51.369084Z","iopub.status.idle":"2022-06-11T01:10:21.231115Z","shell.execute_reply.started":"2022-06-11T01:08:51.369035Z","shell.execute_reply":"2022-06-11T01:10:21.229778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not USE_MADE_FEATURES:\n    df_orders = pd.read_csv(\n        f'{DATA_PATH}train_orders.csv',\n        index_col='id',\n        squeeze=True,\n    ).str.split()","metadata":{"id":"lGM4gXl4-Xm5","execution":{"iopub.status.busy":"2022-06-11T01:10:21.233330Z","iopub.execute_input":"2022-06-11T01:10:21.233868Z","iopub.status.idle":"2022-06-11T01:10:24.205743Z","shell.execute_reply.started":"2022-06-11T01:10:21.233817Z","shell.execute_reply":"2022-06-11T01:10:24.204737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not USE_MADE_FEATURES:\n    df_orders_ = df_orders.to_frame().join(\n        df.reset_index('cell_id').groupby('id')['cell_id'].apply(list),\n        how='right',\n    )","metadata":{"id":"L6oPZoqRN6Y0","execution":{"iopub.status.busy":"2022-06-11T01:10:24.207612Z","iopub.execute_input":"2022-06-11T01:10:24.208704Z","iopub.status.idle":"2022-06-11T01:10:34.998791Z","shell.execute_reply.started":"2022-06-11T01:10:24.208622Z","shell.execute_reply":"2022-06-11T01:10:34.997313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not USE_MADE_FEATURES:\n    ranks = {}\n    for id_, cell_order, cell_id in df_orders_.itertuples():\n        ranks[id_] = {'cell_id': cell_id, 'rank': get_ranks(cell_order, cell_id)}\n    df_ranks = (\n        pd.DataFrame\n            .from_dict(ranks, orient='index')\n            .rename_axis('id')\n            .apply(pd.Series.explode)\n            .set_index('cell_id', append=True)\n    )","metadata":{"id":"Lc3FA00a5P7G","execution":{"iopub.status.busy":"2022-06-11T01:10:35.003981Z","iopub.execute_input":"2022-06-11T01:10:35.004395Z","iopub.status.idle":"2022-06-11T01:11:55.663271Z","shell.execute_reply.started":"2022-06-11T01:10:35.004362Z","shell.execute_reply":"2022-06-11T01:11:55.661916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not USE_MADE_FEATURES:\n    df_ancestors = pd.read_csv(f'{DATA_PATH}train_ancestors.csv', index_col='id')\n    df = df.reset_index().merge(df_ranks, on=[\"id\", \"cell_id\"]).merge(df_ancestors, on=[\"id\"])\n    df[\"pct_rank\"] = df[\"rank\"] / df.groupby(\"id\")[\"cell_id\"].transform(\"count\")","metadata":{"id":"Zt3r7lZu5P7G","execution":{"iopub.status.busy":"2022-06-11T01:11:55.664740Z","iopub.execute_input":"2022-06-11T01:11:55.665106Z","iopub.status.idle":"2022-06-11T01:12:32.009701Z","shell.execute_reply.started":"2022-06-11T01:11:55.665073Z","shell.execute_reply":"2022-06-11T01:12:32.008491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 데이타 분할","metadata":{"id":"Jb51NAl65P7G"}},{"cell_type":"code","source":"if not USE_MADE_FEATURES:\n    train_data = cv_split(df, N_FOLD)\n    display(train_data)","metadata":{"id":"iNx0rP5I5P7G","execution":{"iopub.status.busy":"2022-06-11T01:12:32.011276Z","iopub.execute_input":"2022-06-11T01:12:32.011888Z","iopub.status.idle":"2022-06-11T01:12:46.769991Z","shell.execute_reply.started":"2022-06-11T01:12:32.011831Z","shell.execute_reply":"2022-06-11T01:12:46.768911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 기본 전처리 데이타 저장","metadata":{"id":"17rZ6Ypq7Z0Z"}},{"cell_type":"code","source":"if not USE_MADE_FEATURES:\n    for i in range(N_FOLD):\n        train_df = train_data[train_data.model_fold_num != i]\n        val_df = train_data[train_data.model_fold_num == i]\n        \n        train_df_mark = train_df[train_df[\"cell_type\"] == \"markdown\"].reset_index(drop=True)\n        val_df_mark = val_df[val_df[\"cell_type\"] == \"markdown\"].reset_index(drop=True)\n        train_df_mark.to_csv(f\"{CLEANED_PATH}train_mark_{i}.csv\", index=False)\n        val_df_mark.to_csv(f\"{CLEANED_PATH}val_mark_{i}.csv\", index=False)\n        val_df.to_csv(f\"{CLEANED_PATH}val.csv\", index=False)\n        train_df.to_csv(f\"{CLEANED_PATH}train.csv\", index=False)","metadata":{"id":"YZo1xciR43s8","execution":{"iopub.status.busy":"2022-06-11T01:19:00.511110Z","iopub.execute_input":"2022-06-11T01:19:00.511775Z","iopub.status.idle":"2022-06-11T01:32:48.948325Z","shell.execute_reply.started":"2022-06-11T01:19:00.511734Z","shell.execute_reply":"2022-06-11T01:32:48.946069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_mark.shape, train_df.shape, train_df_mark[train_df_mark[\"cell_type\"] == \"markdown\"].shape","metadata":{"execution":{"iopub.status.busy":"2022-06-11T01:52:23.390988Z","iopub.execute_input":"2022-06-11T01:52:23.391453Z","iopub.status.idle":"2022-06-11T01:52:24.593049Z","shell.execute_reply.started":"2022-06-11T01:52:23.391416Z","shell.execute_reply":"2022-06-11T01:52:24.592165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 추가 전처리 데이타 저장","metadata":{"id":"F3Jkb_CA5P7H"}},{"cell_type":"code","source":"%%time\nif not USE_MADE_FEATURES:\n    for i in range(N_FOLD):\n        train_df = train_data[train_data.model_fold_num != i]\n        val_df = train_data[train_data.model_fold_num == i]\n        \n        val_fts = get_features(val_df)\n        json.dump(val_fts, open(f\"{CLEANED_PATH}val_fts_{i}.json\", \"wt\"))\n        train_fts = get_features(train_df)\n        json.dump(train_fts, open(f\"{CLEANED_PATH}train_fts_{i}.json\",\"wt\"))","metadata":{"id":"R2MivXXsOqGP","execution":{"iopub.status.busy":"2022-06-11T01:32:48.951604Z","iopub.execute_input":"2022-06-11T01:32:48.952269Z","iopub.status.idle":"2022-06-11T01:51:32.822590Z","shell.execute_reply.started":"2022-06-11T01:32:48.952233Z","shell.execute_reply":"2022-06-11T01:51:32.821497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# End Of Program","metadata":{"id":"V8apgTwUH21Q"}}]}