{"metadata":{"accelerator":"GPU","colab":{"provenance":[],"gpuType":"T4"},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Train","metadata":{"id":"UAUMXSkrv8gG"}},{"cell_type":"markdown","source":"## Config","metadata":{"id":"Xa3yTijkIIGt"}},{"cell_type":"code","source":"FOLDERNAME = '/3-aikuthon'\nassert FOLDERNAME is not None, \"[!] Enter the foldername.\"\n\nimport sys\nsys.path.append(f'/content/drive/My Drive/{FOLDERNAME}')","metadata":{"id":"BBdGFPCXvTTG","outputId":"12ec36d7-d070-45bd-daf8-989b7dcf989f","execution":{"iopub.status.busy":"2023-08-25T11:32:40.955005Z","iopub.execute_input":"2023-08-25T11:32:40.955779Z","iopub.status.idle":"2023-08-25T11:32:40.961467Z","shell.execute_reply.started":"2023-08-25T11:32:40.955741Z","shell.execute_reply":"2023-08-25T11:32:40.960264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport numpy as np\n\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom sklearn.model_selection import train_test_split\nimport pandas as pd\n\nimport random\nimport os\nfrom tqdm import tqdm, trange\n\n## Seed 설정, 하는 이유는 https://m.blog.naver.com/vail131/222306329719 참조\nrandom.seed(42)\nnp.random.seed(42)\ntorch.manual_seed(42)\ntorch.cuda.manual_seed(42)","metadata":{"id":"nTt_JMw0vv31","execution":{"iopub.status.busy":"2023-08-25T11:32:41.168663Z","iopub.execute_input":"2023-08-25T11:32:41.169074Z","iopub.status.idle":"2023-08-25T11:32:41.177373Z","shell.execute_reply.started":"2023-08-25T11:32:41.169045Z","shell.execute_reply":"2023-08-25T11:32:41.176032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install transformers","metadata":{"id":"oQb2m2xcfugs","outputId":"aef275e2-2249-4dfb-985d-541e04c39fa4","execution":{"iopub.status.busy":"2023-08-25T11:32:41.330950Z","iopub.execute_input":"2023-08-25T11:32:41.331343Z","iopub.status.idle":"2023-08-25T11:32:53.632313Z","shell.execute_reply.started":"2023-08-25T11:32:41.331312Z","shell.execute_reply":"2023-08-25T11:32:53.630927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Prepeocessing","metadata":{"id":"5BF2MnrpH7ko"}},{"cell_type":"code","source":"CSV_PATH = f'/kaggle/input/{FOLDERNAME}/train.csv'\ndf = pd.read_csv(CSV_PATH)\n\n# Give id column\ndf = df.rename(columns=({'Unnamed: 0':'id'}))\nprint(df.shape)\ndf","metadata":{"id":"5v88p5aKauGQ","outputId":"3485f6ef-cd86-4783-f144-aecc73d928b1","execution":{"iopub.status.busy":"2023-08-25T12:04:48.479680Z","iopub.execute_input":"2023-08-25T12:04:48.480062Z","iopub.status.idle":"2023-08-25T12:04:51.715202Z","shell.execute_reply.started":"2023-08-25T12:04:48.480032Z","shell.execute_reply":"2023-08-25T12:04:51.714030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\n## 단어 수 plot\nword_count = df['lyrics'].apply(lambda x: len(x.split()))\nfig = plt.figure(figsize=[10, 7])\nsns.distplot(word_count, color=sns.xkcd_rgb['greenish teal'])\nplt.show()","metadata":{"id":"Ffd4F-niDlkv","outputId":"eabe47b9-5d49-41f2-c5c6-ef6e51faffb3","execution":{"iopub.status.busy":"2023-08-25T12:04:51.717818Z","iopub.execute_input":"2023-08-25T12:04:51.718686Z","iopub.status.idle":"2023-08-25T12:04:54.651324Z","shell.execute_reply.started":"2023-08-25T12:04:51.718641Z","shell.execute_reply":"2023-08-25T12:04:54.650351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 장르 수\ngenre_counts = df['genre'].value_counts()\ngenre_counts.sort_values(ascending=False)\nfor index, value in genre_counts.items(): print(index, value)","metadata":{"id":"TKEmaD3HTMhB","outputId":"e3a21102-b05a-4d38-8252-25aacf6d7072","execution":{"iopub.status.busy":"2023-08-25T12:04:54.652749Z","iopub.execute_input":"2023-08-25T12:04:54.653323Z","iopub.status.idle":"2023-08-25T12:04:54.679683Z","shell.execute_reply.started":"2023-08-25T12:04:54.653286Z","shell.execute_reply":"2023-08-25T12:04:54.678554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## index와 label간의 대응 관계를 id2label, label2id로 정의합니다.\nid2label = {'발라드': 0,\n '록/메탈': 1,\n '댄스': 2,\n '포크/블루스': 3,\n 'POP': 4,\n 'R&B/Soul': 5,\n '-': 6,\n '랩/힙합': 7,\n '국내영화': 8,\n 'CCM': 9,\n '국내CCM': 10,\n '클래식': 11,\n '오페라/성악': 12,\n '국외영화': 13,\n '국내드라마': 14,\n '인디음악': 15,\n '성인가요/트로트': 16,\n '일렉트로니카': 17,\n '뉴에이지': 18,\n '키즈': 19,\n '창작동요': 20,\n '크로스오버': 21,\n '재즈': 22,\n '보컬재즈': 23,\n '애시드/퓨전/팝': 24,\n '월드뮤직': 25,\n '애니메이션/웹툰': 26,\n '만화': 27,\n '게임': 28,\n 'J-POP': 29,\n '중국음악': 30,\n '샹송/프렌치팝': 31,\n '컨트리': 32,\n '국내뮤지컬': 33,\n '블루스': 34,\n '국외CCM': 35,\n '포크': 36,\n '국외뮤지컬': 37,\n '가톨릭': 38,\n '국외드라마': 39,\n '브라질': 40,\n '뮤직테라피': 41,\n '보사노바': 42,\n '라틴': 43,\n '국악': 44,\n '국악가요': 45,\n '찬송가': 46,\n '영어동요': 47,\n '워십': 48,\n '불교': 49,\n '자장가': 50,\n '민요': 51}\n\nlabel2id = {value: key for key, value in id2label.items()}\n\n## num_classes를 정의합니다.\nnum_classes = len(id2label)\n\nprint(f\"num_classes: {num_classes}\")\nprint(f\"id2label: {id2label}\")\nprint(f\"label2id: {label2id}\")\n\n","metadata":{"id":"_4IMxRRVTL4t","outputId":"80736c46-c90c-43ba-95a8-c133568cb02c","execution":{"iopub.status.busy":"2023-08-25T12:06:06.487562Z","iopub.execute_input":"2023-08-25T12:06:06.488571Z","iopub.status.idle":"2023-08-25T12:06:06.502205Z","shell.execute_reply.started":"2023-08-25T12:06:06.488535Z","shell.execute_reply":"2023-08-25T12:06:06.500882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 장르 필터링\nsplitted_genre_df = df.copy()\nsplitted_genre_df[\"genre\"] = df['genre'].str.split(\",\")\nsplitted_genre_df[\"genre\"] = splitted_genre_df[\"genre\"].apply(lambda x : list(map(int, x)))\n\n### df에서 여러개의 장르가 함께 들어있는 행을 제외합니다.\n# filtered_df = df[~df['genre'].str.contains(\",\")]\n\ndef df_only_specific_genre(df, genre):\n    res = []\n    genre_id = id2label[genre]\n    for i, data in enumerate(df['genre']):\n        res.append(genre_id in data)\n    \n    filtered_df = df.copy()\n    filtered_df = filtered_df[res]\n    filtered_df[\"genre\"] = genre_id\n    return filtered_df\n\n### filtered_df에서 원하는 장르만 가져옵니다.\nballad = df_only_specific_genre(splitted_genre_df, \"발라드\")\nrock = df_only_specific_genre(splitted_genre_df, '록/메탈')\ndance = df_only_specific_genre(splitted_genre_df, '댄스')\npop = df_only_specific_genre(splitted_genre_df, 'POP')\nhiphop = df_only_specific_genre(splitted_genre_df, '랩/힙합')\nrnb = df_only_specific_genre(splitted_genre_df, 'R&B/Soul')\ntrot = df_only_specific_genre(splitted_genre_df, '성인가요/트로트')\n\n\nprint(len(ballad), len(rock), len(dance), len(pop), len(rnb), len(hiphop), len(trot))\nfiltered_df = pd.concat([ballad, rock, dance, pop, rnb, hiphop, trot])\n\n\n\n# ballad = splitted_genre_df[str() in splitted_genre_df['genre']]\n# splitted_genre_df","metadata":{"id":"fSwjTksLXOgb","outputId":"0255b8f7-28db-4e2e-b052-fddeaf2231f1","execution":{"iopub.status.busy":"2023-08-25T12:06:06.693952Z","iopub.execute_input":"2023-08-25T12:06:06.694609Z","iopub.status.idle":"2023-08-25T12:06:08.322836Z","shell.execute_reply.started":"2023-08-25T12:06:06.694570Z","shell.execute_reply":"2023-08-25T12:06:08.321600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 필터링 후 장르 수 확인\nprint(len(filtered_df))\ngenre_counts = filtered_df['genre'].value_counts()\ngenre_counts.sort_values(ascending=False)\nfor index, value in genre_counts.items(): print(index, value)","metadata":{"id":"5-bVgL0ZmVjy","outputId":"380d63ec-d2a9-4f94-c5ca-df7fa246aa3f","execution":{"iopub.status.busy":"2023-08-25T12:06:10.721727Z","iopub.execute_input":"2023-08-25T12:06:10.722239Z","iopub.status.idle":"2023-08-25T12:06:10.733572Z","shell.execute_reply.started":"2023-08-25T12:06:10.722185Z","shell.execute_reply":"2023-08-25T12:06:10.732408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Data Sampling\n## 1. 각 장르별로 sample 수 같도록 샘플링\n# num_samples_per_group = 7441\n\n# grouped = filtered_df.groupby('genre').apply(lambda group: group.sample(n=num_samples_per_group))\n# random_sample_df = grouped.reset_index(drop=True)\n\n# sampled_indices = random_sample_df.index\n\n# genre_categories = [str(id2label[id]) for id in ('POP', 'R&B/Soul', '댄스', '랩/힙합', '록/메탈', '발라드', '성인가요/트로트')]\n# random_sample_df['genre'] = pd.Categorical(random_sample_df['genre'], categories=genre_categories, ordered=True)\n\n# print(random_sample_df['genre'].unique())\n\n## 2. 전부 사용\nrandom_sample_df = filtered_df.reset_index(drop=True)\n\nrandom_sample_df","metadata":{"id":"uXAlEnucFW4x","outputId":"e894adcf-019c-43c4-8533-c81de029f615","execution":{"iopub.status.busy":"2023-08-25T12:06:11.588831Z","iopub.execute_input":"2023-08-25T12:06:11.589220Z","iopub.status.idle":"2023-08-25T12:06:11.639580Z","shell.execute_reply.started":"2023-08-25T12:06:11.589183Z","shell.execute_reply":"2023-08-25T12:06:11.638449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SongDataset(Dataset):\n    def __init__(self, df, id2label):\n        df = df.reset_index(drop=True)\n        self.song_names = df[\"song_name\"]\n        self.artists = df[\"artist\"]\n        self.lyrics = df[\"lyrics\"]\n        self.genres = df[\"genre\"]\n        self.id = df[\"id\"]\n        self.id2label = id2label\n        self.date = df[\"date\"]\n\n    def __getitem__(self, idx):\n        date = self.date[idx].split(\".\")\n        if len(date) < 3: year, month = '없', '음'\n        else: year, month = date[0], f\" {date[1]}\"\n            \n        song_name = self.song_names[idx]\n        artist = self.artists[idx]\n        lyrics = f\"{self.song_names[idx]} [SEP] {self.artists[idx]} [SEP] {year}{month} [SEP] {self.lyrics[idx]}\"\n        id = self.id[idx]\n        genre = int(self.genres[idx])\n        genre = torch.Tensor([1 if i == genre else 0 for i in range(len(self.id2label))])\n\n        return {'song_name': song_name,\n                'artist': artist,\n                'lyrics': lyrics,\n                'genre': genre,\n                'id': id}\n\n    def __len__(self):\n        return len(self.song_names)","metadata":{"id":"VnY0AADPmKY1","execution":{"iopub.status.busy":"2023-08-25T12:06:12.265730Z","iopub.execute_input":"2023-08-25T12:06:12.266562Z","iopub.status.idle":"2023-08-25T12:06:12.285369Z","shell.execute_reply.started":"2023-08-25T12:06:12.266515Z","shell.execute_reply":"2023-08-25T12:06:12.283608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Dataset 체크\ndataset_for_check = SongDataset(random_sample_df, id2label)\ndataset_for_check[0]","metadata":{"id":"gx15i_6ivzo7","outputId":"234d82d0-03d1-4b70-ea55-be4d45960bde","execution":{"iopub.status.busy":"2023-08-25T12:06:13.087889Z","iopub.execute_input":"2023-08-25T12:06:13.088269Z","iopub.status.idle":"2023-08-25T12:06:13.141725Z","shell.execute_reply.started":"2023-08-25T12:06:13.088240Z","shell.execute_reply":"2023-08-25T12:06:13.140547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{"id":"OJR2vR4jIAZH"}},{"cell_type":"code","source":"## Model & Training Configurations\nMAX_LEN_TRAIN = 128\nMAX_LEN_VALID = 128\n\nBATCH_SIZE = 64\nLR = 1e-4\nNUM_EPOCHS = 13\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nprint('using device:', device)\n\nMODEL_OUT_DIR = '/content/drive/My Drive/{}/models/KRBERT-classifier'.format(FOLDERNAME)\nif not os.path.isdir(MODEL_OUT_DIR):\n    os.makedirs(MODEL_OUT_DIR)","metadata":{"id":"tiUP8zSZCaaL","outputId":"0b1596dd-9d8f-4fab-c1f2-95b635ab2319","execution":{"iopub.status.busy":"2023-08-25T14:37:05.511713Z","iopub.execute_input":"2023-08-25T14:37:05.512162Z","iopub.status.idle":"2023-08-25T14:37:05.521704Z","shell.execute_reply.started":"2023-08-25T14:37:05.512129Z","shell.execute_reply":"2023-08-25T14:37:05.520647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import BertTokenizer, BertForSequenceClassification, DistilBertTokenizerFast\n\nMODEL_NAME = 'snunlp/KR-Medium'\n\ntokenizer = DistilBertTokenizerFast.from_pretrained(MODEL_NAME, do_lower_case=False)\nmodel = BertForSequenceClassification.from_pretrained(MODEL_NAME, num_labels=num_classes).to(device)","metadata":{"id":"IAebJrjpIX-W","outputId":"d0539883-a020-4361-e8b5-6f5494bd04fa","execution":{"iopub.status.busy":"2023-08-25T14:37:11.462497Z","iopub.execute_input":"2023-08-25T14:37:11.465196Z","iopub.status.idle":"2023-08-25T14:37:13.132610Z","shell.execute_reply.started":"2023-08-25T14:37:11.465152Z","shell.execute_reply":"2023-08-25T14:37:13.131601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tokenizer test\nprint(tokenizer(\"듣고있는데 재밌네요\"))\nprint(tokenizer(\"듣고있는데 재밌네요\")['input_ids'])\n[tokenizer.decode(i) for i in tokenizer(\"듣고있는데 재밌네요\")['input_ids']]","metadata":{"id":"fEGKRcnDB9PQ","outputId":"a4490ce3-8efb-4436-ed1c-5696e533df80","execution":{"iopub.status.busy":"2023-08-25T14:37:15.554281Z","iopub.execute_input":"2023-08-25T14:37:15.554740Z","iopub.status.idle":"2023-08-25T14:37:15.565879Z","shell.execute_reply.started":"2023-08-25T14:37:15.554706Z","shell.execute_reply":"2023-08-25T14:37:15.564625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\noptimizer = optim.AdamW(model.parameters(), lr=LR, weight_decay=1e-5)","metadata":{"id":"pnzObGbnoLzs","execution":{"iopub.status.busy":"2023-08-25T14:37:19.442851Z","iopub.execute_input":"2023-08-25T14:37:19.443903Z","iopub.status.idle":"2023-08-25T14:37:19.451480Z","shell.execute_reply.started":"2023-08-25T14:37:19.443862Z","shell.execute_reply":"2023-08-25T14:37:19.450125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training & Evaluation","metadata":{"id":"QZvhHuRSCjxb"}},{"cell_type":"code","source":"def train(model, criterion, optimizer, train_loader, val_loader, num_epochs, device):\n\n  ## val accuracy가 가장 높은 모델을 저장\n  best_val_accuracy = 0.\n  global best_model\n\n  for iter in range(1, num_epochs+1):\n    print(f\"=====================epoch {iter}=====================\")\n    total_loss = 0.0\n    correct_predictions = 0\n    incorrect_predictions = 0\n    model.train()\n    with tqdm(train_loader, unit=\"batch\") as tepoch:\n        for data in tepoch:\n            \n            inputs, labels = data[\"lyrics\"], data[\"genre\"].float().to(device)\n            encoded_input = tokenizer(inputs, return_tensors='pt', padding='max_length', truncation=True, max_length=MAX_LEN_TRAIN).to(device)\n            outputs = model(**encoded_input).logits\n\n            # loss backpropagation\n            loss = criterion(outputs, labels)\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n            # accuracy\n            _, predicted = torch.max(outputs, 1)\n            _, sol = torch.max(labels, 1)\n            correct_predictions += (predicted == sol).sum().item()\n            incorrect_predictions += BATCH_SIZE - (predicted == sol).sum().item()\n\n            # update tqdm\n            tepoch.set_postfix(loss=loss.item(), accuracy= correct_predictions / (correct_predictions + incorrect_predictions))\n            total_loss += loss.item()\n\n\n    print(f\"Epoch {iter}/{num_epochs}, Train Loss: {total_loss / len(train_dataloader)}, \"\n          f\"Training Accuracy: {correct_predictions / (incorrect_predictions+ correct_predictions)}\")\n\n    # Validation 진행\n    model.eval()\n    val_loss = 0.0\n    val_correct_predictions = 0\n    val_incorrect_predictions = 0\n    with torch.no_grad():\n        with tqdm(val_loader, unit=\"batch\") as tepoch:\n            for data in tepoch:\n                \n                inputs, labels = data[\"lyrics\"], data[\"genre\"].float().to(device)\n                encoded_input = tokenizer(inputs, return_tensors='pt', padding='max_length', truncation=True, max_length=MAX_LEN_VALID).to(device)\n                outputs = model(**encoded_input).logits\n\n                # loss (only calculation)\n                loss = criterion(outputs, labels)\n                val_loss += loss.item()\n\n                # accuracy\n                _, predicted = torch.max(outputs, 1)\n                _, sol = torch.max(labels, 1)\n                val_correct_predictions += (predicted == sol).sum().item()\n                val_incorrect_predictions += BATCH_SIZE - (predicted == sol).sum().item()\n\n                # update tqdm\n                tepoch.set_postfix(loss=loss.item(), accuracy= val_correct_predictions / (val_correct_predictions + val_incorrect_predictions))\n                total_loss += loss.item()\n\n    val_accuracy = val_correct_predictions / (val_incorrect_predictions + val_correct_predictions)\n    print(f\"Epoch {iter}/{num_epochs}, Validation Loss: {val_loss / len(val_dataloader)}, \"\n          f\"Validation Accuracy: {val_accuracy}\", end=\"\")\n\n    # save best model\n    if best_val_accuracy < val_accuracy:\n        best_model = model\n        best_val_accuracy = val_accuracy\n        print(\" best model upadated.\")\n    else:\n        print()","metadata":{"id":"LgAZwWLYv1sq","execution":{"iopub.status.busy":"2023-08-25T14:37:21.161947Z","iopub.execute_input":"2023-08-25T14:37:21.162364Z","iopub.status.idle":"2023-08-25T14:37:21.183806Z","shell.execute_reply.started":"2023-08-25T14:37:21.162332Z","shell.execute_reply":"2023-08-25T14:37:21.182576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Dataset\ndatasets = SongDataset(random_sample_df, id2label)\n\n## train set과 validation set을 정해진 비율 (여기서는 8:2)로 나눕니다.\ntrain_len = int(len(datasets) * 0.9)\nval_len = len(datasets) - train_len\nprint(train_len, val_len)\ntrain_dataset, val_dataset = random_split(datasets, [train_len, val_len])\n\n## Dataloder\ntrain_dataloader = DataLoader(train_dataset, batch_size = BATCH_SIZE, shuffle = True)\nval_dataloader = DataLoader(val_dataset, batch_size = BATCH_SIZE, shuffle = True)","metadata":{"id":"hTPKMDk_CgNz","outputId":"e75038b8-d6e0-424d-cc42-6e467087bd55","execution":{"iopub.status.busy":"2023-08-25T14:37:22.851910Z","iopub.execute_input":"2023-08-25T14:37:22.852310Z","iopub.status.idle":"2023-08-25T14:37:22.879277Z","shell.execute_reply.started":"2023-08-25T14:37:22.852277Z","shell.execute_reply":"2023-08-25T14:37:22.878158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 재실행시 CUDA out of memory 방지용\nimport gc\ngc.collect()\ntorch.cuda.empty_cache()\n\n## train, eval\ntrain(model=model,\n      criterion=criterion,\n      optimizer=optimizer,\n      train_loader=train_dataloader,\n      val_loader=val_dataloader,\n      num_epochs = 10,\n      device = device)","metadata":{"id":"csfDursdv22u","outputId":"b0d95034-5c8d-49de-80e4-23a73206c14c","execution":{"iopub.status.busy":"2023-08-25T14:37:24.360004Z","iopub.execute_input":"2023-08-25T14:37:24.360451Z","iopub.status.idle":"2023-08-25T17:15:02.432601Z","shell.execute_reply.started":"2023-08-25T14:37:24.360408Z","shell.execute_reply":"2023-08-25T17:15:02.431244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime\nnow = datetime.now()\n\n## Save bset model\nMODEL_PATH = f'/kaggle/working/best_model_{int(now.timestamp())}.pt'\ntorch.save(best_model.state_dict(), MODEL_PATH)","metadata":{"id":"EdINIGTKv3-V","execution":{"iopub.status.busy":"2023-08-25T17:15:04.695133Z","iopub.execute_input":"2023-08-25T17:15:04.698272Z","iopub.status.idle":"2023-08-25T17:15:05.234398Z","shell.execute_reply.started":"2023-08-25T17:15:04.698230Z","shell.execute_reply":"2023-08-25T17:15:05.233233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{"id":"eKdQwW-sv6U-"}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport numpy as np\n\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom sklearn.model_selection import train_test_split\nimport pandas as pd\n\nimport random\nimport os\nfrom tqdm import tqdm, trange\nfrom pprint import pprint","metadata":{"id":"_cZosjvHyX4t","execution":{"iopub.status.busy":"2023-08-25T17:15:05.236326Z","iopub.execute_input":"2023-08-25T17:15:05.236689Z","iopub.status.idle":"2023-08-25T17:15:05.246604Z","shell.execute_reply.started":"2023-08-25T17:15:05.236660Z","shell.execute_reply":"2023-08-25T17:15:05.244528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Seed 설정, 하는 이유는 https://m.blog.naver.com/vail131/222306329719 참조\nrandom.seed(42)\nnp.random.seed(42)\ntorch.manual_seed(42)\ntorch.cuda.manual_seed(42)","metadata":{"id":"T3LFkMbUw069","execution":{"iopub.status.busy":"2023-08-25T17:15:05.321879Z","iopub.execute_input":"2023-08-25T17:15:05.322172Z","iopub.status.idle":"2023-08-25T17:15:05.327126Z","shell.execute_reply.started":"2023-08-25T17:15:05.322146Z","shell.execute_reply":"2023-08-25T17:15:05.326190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 모델 불러오기\n\n### '.pt등의 파일로 저장되어있는' 모델을 불러올 경우\n# from transformers import BertTokenizer, BertForSequenceClassification, DistilBertTokenizerFast\n\n# device = \"cuda\"\n# MODEL_PATH = \"/kaggle/working/best_model_1692957252.pt\" ## model weight path\n# tokenizer = DistilBertTokenizerFast.from_pretrained('snunlp/KR-Medium', do_lower_case=False) ## model used in training phase\n# test_model = BertForSequenceClassification.from_pretrained('snunlp/KR-Medium', num_labels=num_classes).to(device)  ## tokenizer used in training phase\n\n\n# test_model.load_state_dict(torch.load(MODEL_PATH))\n\n# test_model.to(device)\n\n### 위에서 훈련시켜 '변수에 저장되어있는' 모델을 사용할 경우\ntest_model = best_model","metadata":{"id":"-5XPVlMJvRZW","outputId":"cc44f3a5-e3ab-4ba0-aa61-66f6d71a6923","execution":{"iopub.status.busy":"2023-08-25T17:15:08.838229Z","iopub.execute_input":"2023-08-25T17:15:08.838940Z","iopub.status.idle":"2023-08-25T17:15:08.844689Z","shell.execute_reply.started":"2023-08-25T17:15:08.838897Z","shell.execute_reply":"2023-08-25T17:15:08.843621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Dataset Config\nTEST_CSV_PATH = f'/kaggle/input/3-aikuthon/test.csv'\nBATCH_SIZE_TEST = 256\n\n\n## genre에 저장되어있는 숫자와 실제 장르 명의 대응관계입니다.\nid2label = {'발라드': 0,\n '록/메탈': 1,\n '댄스': 2,\n '포크/블루스': 3,\n 'POP': 4,\n 'R&B/Soul': 5,\n '-': 6,\n '랩/힙합': 7,\n '국내영화': 8,\n 'CCM': 9,\n '국내CCM': 10,\n '클래식': 11,\n '오페라/성악': 12,\n '국외영화': 13,\n '국내드라마': 14,\n '인디음악': 15,\n '성인가요/트로트': 16,\n '일렉트로니카': 17,\n '뉴에이지': 18,\n '키즈': 19,\n '창작동요': 20,\n '크로스오버': 21,\n '재즈': 22,\n '보컬재즈': 23,\n '애시드/퓨전/팝': 24,\n '월드뮤직': 25,\n '애니메이션/웹툰': 26,\n '만화': 27,\n '게임': 28,\n 'J-POP': 29,\n '중국음악': 30,\n '샹송/프렌치팝': 31,\n '컨트리': 32,\n '국내뮤지컬': 33,\n '블루스': 34,\n '국외CCM': 35,\n '포크': 36,\n '국외뮤지컬': 37,\n '가톨릭': 38,\n '국외드라마': 39,\n '브라질': 40,\n '뮤직테라피': 41,\n '보사노바': 42,\n '라틴': 43,\n '국악': 44,\n '국악가요': 45,\n '찬송가': 46,\n '영어동요': 47,\n '워십': 48,\n '불교': 49,\n '자장가': 50,\n '민요': 51}\nlabel2id = {value: key for key, value in id2label.items()}\n\nnum_classes = len(id2label)\nprint(f\"num_classes: {num_classes}\")","metadata":{"id":"VmmQ8lNFf648","outputId":"a57fc0fa-687f-40c9-948c-96b5554945f6","execution":{"iopub.status.busy":"2023-08-25T17:15:11.649480Z","iopub.execute_input":"2023-08-25T17:15:11.650376Z","iopub.status.idle":"2023-08-25T17:15:11.662910Z","shell.execute_reply.started":"2023-08-25T17:15:11.650340Z","shell.execute_reply":"2023-08-25T17:15:11.661704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## test df를 불러옵니다.\ntest_df = pd.read_csv(TEST_CSV_PATH)\n\n## 첫 열의 이름을 'id'로 바꾸어줍니다.\ntest_df = test_df.rename(columns=({'Unnamed: 0':'id'}))\nprint(test_df.shape)\ntest_df","metadata":{"id":"UTDLiWCQsaPw","outputId":"ac56b3ef-9adf-4dfc-a4ce-f33a90fa8e96","execution":{"iopub.status.busy":"2023-08-25T17:15:12.084636Z","iopub.execute_input":"2023-08-25T17:15:12.085025Z","iopub.status.idle":"2023-08-25T17:15:12.399676Z","shell.execute_reply.started":"2023-08-25T17:15:12.084995Z","shell.execute_reply":"2023-08-25T17:15:12.398634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## define dataset class\nclass SongDatasetTest(Dataset):\n    def __init__(self, df):\n        df = df.reset_index(drop=True)\n        self.song_names = df[\"song_name\"]\n        self.artists = df[\"artist\"]\n        self.lyrics = df[\"lyrics\"]\n        self.id = df[\"id\"]\n\n    def __getitem__(self, idx):\n        song_name = self.song_names[idx]\n        artist = self.artists[idx]\n        lyrics = f\"{self.song_names[idx]} [SEP] {self.artists[idx]} [SEP] {self.lyrics[idx]}\"\n        id_ = self.id[idx]\n\n        return {'song_name': song_name,\n                'artist': artist,\n                'lyrics': lyrics,\n                'id': id_}\n\n    def __len__(self):\n        return len(self.song_names)","metadata":{"id":"40Hw_i-OsdUJ","execution":{"iopub.status.busy":"2023-08-25T17:15:12.419255Z","iopub.execute_input":"2023-08-25T17:15:12.420285Z","iopub.status.idle":"2023-08-25T17:15:12.428270Z","shell.execute_reply.started":"2023-08-25T17:15:12.420248Z","shell.execute_reply":"2023-08-25T17:15:12.426707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## define test_dataset, dataloader\ntest_datasets = SongDatasetTest(test_df)\n\n## shuffle=False로 설정해야 출력 순서가 바뀌지 않아 제대로 된 채점이 가능합니다.\ntest_dataloader = DataLoader(test_datasets, batch_size = BATCH_SIZE_TEST, shuffle = False)\ntest_datasets[5515]","metadata":{"id":"0tOa5GH6srTj","outputId":"dce2512e-a65e-447d-e622-77f926219ee8","execution":{"iopub.status.busy":"2023-08-25T17:15:14.155276Z","iopub.execute_input":"2023-08-25T17:15:14.155961Z","iopub.status.idle":"2023-08-25T17:15:14.168507Z","shell.execute_reply.started":"2023-08-25T17:15:14.155926Z","shell.execute_reply":"2023-08-25T17:15:14.167439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## predict\n## 확률이 가장 높게 예측된 index를 result에 모두 저장합니다.\nimport gc\ngc.collect()\ntorch.cuda.empty_cache()\nMAX_LEN_TEST = 64\nprint(device)\nresult = []\nwith torch.no_grad():\n  test_correct_predictions, test_incorrect_predictions, count = 0, 0, 1\n  for data in test_dataloader:\n        \n#     inputs, labels = data[\"song_name\"] + '\\n' + data[\"lyrics\"], data[\"genre\"].float().to(device)\n\n#     encoded_input = tokenizer(artist, text_pair = inputs, return_tensors='pt', padding='max_length', truncation=True, max_length=MAX_LEN_TEST).to(device)\n\n    inputs = data['lyrics']\n    encoded_input = tokenizer(inputs, return_tensors='pt', padding='max_length', truncation=True, max_length=MAX_LEN_TEST).to(device)\n    test_model = test_model.to(device)\n\n    test_model.eval()\n    outputs = test_model(**encoded_input).logits\n    _, predicted = torch.max(outputs, 1)\n\n    for i, output in enumerate(outputs):\n      song = data['song_name'][i]\n      artist = data['artist'][i]\n      predicted = torch.topk(output, 1).indices.item()\n      id = data['id'][i].item()\n\n      output_for_print = {\n          \"Song\": song,\n          \"Artist\": artist,\n          \"Predicted\": label2id[predicted],\n          \"Id\": str(id)\n      }\n\n      output_for_submission = {\n          \"id\": id,\n          \"genre\": predicted\n      }\n\n      result.append(output_for_submission)\n      pprint(output_for_print, sort_dicts=False)\n    print(f\"{count} / {len(test_datasets)}\")\n    count += len(outputs)\n      ","metadata":{"id":"4_J20OgpdU9o","outputId":"96c73cd6-6b47-4195-a584-93319f5dcd49","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 예측된 값의 리스트를 DataFrame 형식의 df_test로 변환하고, csv file로 저장합니다.\ndf_test = pd.DataFrame(result)\nRESULT_PATH = f'/kaggle/working/prediction11.csv'\ndf_test.to_csv(RESULT_PATH, index=False)\ndf_test","metadata":{"id":"PIKnvYOo9zpr","outputId":"c735619c-f32e-417f-9fb5-ff11670aaf85","execution":{"iopub.status.busy":"2023-08-25T17:16:19.812562Z","iopub.execute_input":"2023-08-25T17:16:19.813006Z","iopub.status.idle":"2023-08-25T17:16:19.874125Z","shell.execute_reply.started":"2023-08-25T17:16:19.812972Z","shell.execute_reply":"2023-08-25T17:16:19.872878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}