{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# AML Challenge 2 Baseline Solution\n\nThis notebook is almost based on the Github [respository](https://github.com/daisukelab/dcase2020_task2_variants). \n\nWe replicate the (Variational Auto-Encoder) VAE method for this AML challenge 2 which is a small scale of the official DCASE challenge [task 2](https://dcase.community/challenge2020/task-unsupervised-detection-of-anomalous-sounds). ","metadata":{}},{"cell_type":"markdown","source":"# Required Python Packages\n\nLet's install some python packages used in this notebook.","metadata":{}},{"cell_type":"code","source":"!pip install torchsummary\n!pip install dl-cliche","metadata":{"execution":{"iopub.status.busy":"2022-05-25T19:05:49.619394Z","iopub.execute_input":"2022-05-25T19:05:49.619977Z","iopub.status.idle":"2022-05-25T19:06:15.540022Z","shell.execute_reply.started":"2022-05-25T19:05:49.619887Z","shell.execute_reply":"2022-05-25T19:06:15.538947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To import our own functions, you could upload the .py files in the input folder and do as we do for the \"common.py\" and \"pytorch_common.py\" files.","metadata":{}},{"cell_type":"code","source":"from shutil import copyfile\n\n# copy our file into the working directory (make sure it has .py suffix)\ncopyfile(src = \"../input/dcase2020-task2-baseline/pytorch_common.py\", dst = \"../working/pytorch_common.py\")\ncopyfile(src = \"../input/dcase2020-task2-baseline/common.py\", dst = \"../working/common.py\")\n\n# import all our functions\nimport common as com\nimport pytorch_common\nfrom pytorch_common import *\n\n# public modules\nimport torch\nimport torch.nn.functional as F\nimport torchsummary\nimport pytorch_lightning as pl\nimport random\nimport common as com\nimport librosa\nimport os\nimport glob\nimport csv\nimport re\nimport itertools\nimport sys\nimport path\nimport numpy\nimport seaborn as sns\nfrom torch import nn\nfrom tqdm import tqdm\nfrom sklearn import metrics\nfrom dlcliche.utils import *\nfrom dlcliche.notebook import *\nfrom dlcliche.utils import (\n    sys, random, Path, np, plt, EasyDict,\n    ensure_folder, deterministic_everything,\n)\n","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-05-25T19:06:20.576266Z","iopub.execute_input":"2022-05-25T19:06:20.576545Z","iopub.status.idle":"2022-05-25T19:06:31.71631Z","shell.execute_reply.started":"2022-05-25T19:06:20.576515Z","shell.execute_reply":"2022-05-25T19:06:31.715436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Exploration\n\nHow many samples included?\n\nHow are the raw wave files?\n\nHow are the log mel spectrogram of the wave files?","metadata":{}},{"cell_type":"code","source":"# development dataset\nDATA_ROOT=Path(\"../input/eurecom-aml-2022-challenge-2/dev_data/dev_data\")\ntypes = [t.name for t in sorted(DATA_ROOT.glob('*')) if t.is_dir()]\nprint('Machine types:', types)\n\ndf = pd.DataFrame()\ndf['file'] = sorted(DATA_ROOT.glob('*/*/*.wav'))\ndf['type'] = df.file.map(lambda f: f.parent.parent.name)\ndf['split'] = df.file.map(lambda f: f.parent.name)\ndf['id'] = df.file.map(lambda s: str(s).split('/')[-1].split('_')[-2])\n\nagg = df.groupby(['id', 'split']).agg('count')\nfig = plt.figure(figsize=(12.0, 6.0))\ng = sns.barplot(x=\"id\", y=\"file\", hue=\"split\", data=agg.reset_index())\nplt.xlabel(\"machine id\")\nplt.ylabel(\"number of files\")\nplt.show()\nagg.transpose()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T19:06:31.71832Z","iopub.execute_input":"2022-05-25T19:06:31.719792Z","iopub.status.idle":"2022-05-25T19:06:32.544596Z","shell.execute_reply.started":"2022-05-25T19:06:31.71975Z","shell.execute_reply":"2022-05-25T19:06:32.543898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# evaluation dataset\nDATA_ROOT=Path(\"../input/eurecom-aml-2022-challenge-2/eval_data/eval_data\")\ntypes = [t.name for t in sorted(DATA_ROOT.glob('*')) if t.is_dir()]\nprint('Machine types:', types)\n\ndf = pd.DataFrame()\ndf['file'] = sorted(DATA_ROOT.glob('*/*/*.wav'))\ndf['type'] = df.file.map(lambda f: f.parent.parent.name)\ndf['split'] = df.file.map(lambda f: f.parent.name)\ndf['id'] = df.file.map(lambda s: str(s).split('/')[-1].split('_')[-2])\n\nagg = df.groupby(['id', 'split']).agg('count')\nfig = plt.figure(figsize=(12.0, 6.0))\ng = sns.barplot(x=\"id\", y=\"file\", hue=\"split\", data=agg.reset_index())\nplt.xlabel(\"machine id\")\nplt.ylabel(\"number of files\")\nplt.show()\nagg.transpose()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T19:07:36.192806Z","iopub.execute_input":"2022-05-25T19:07:36.193216Z","iopub.status.idle":"2022-05-25T19:07:37.075095Z","shell.execute_reply.started":"2022-05-25T19:07:36.193169Z","shell.execute_reply":"2022-05-25T19:07:37.074321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_log_mel_spectrogram(filename, n_mels=64,\n                        n_fft=1024,\n                        hop_length=512,\n                        power=2.0):\n    wav, sampling_rate = com.file_load(filename)\n    mel_spectrogram = librosa.feature.melspectrogram(y=wav,\n                                                     sr=sampling_rate,\n                                                     n_fft=n_fft,\n                                                     hop_length=hop_length,\n                                                     n_mels=n_mels,\n                                                     power=power)\n    log_mel_spectrogram = 20.0 / power * np.log10(mel_spectrogram + sys.float_info.epsilon)\n    return log_mel_spectrogram, wav\n\nfor t in types:\n    for split in ['train', 'test']:\n        type_df = df[df['type'] == t][df.split == split].reset_index()\n        R = 2\n        fig, ax = plt.subplots(R, 2, figsize = (15, 1.5*R))\n        print(f'=== Machine type [{t}], {split} set ===')\n        for i in range(R * 2):\n            file_index = i//2\n            file_path = Path(type_df.file[file_index])\n            mels, wav = get_log_mel_spectrogram(file_path)\n            if i%2==0:\n                ax[i//2, i%2].set_title(file_path.name)\n                ax[i//2, i%2].imshow(mels)\n                ax[i//2, i%2].axis('off')\n            else:\n                ax[i//2, i%2].set_title(file_path.name)\n                ax[i//2, i%2].plot(wav)\n                ax[i//2, i%2].get_xaxis().set_ticks([])\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T19:34:34.770715Z","iopub.execute_input":"2022-05-25T19:34:34.771033Z","iopub.status.idle":"2022-05-25T19:34:35.845218Z","shell.execute_reply.started":"2022-05-25T19:34:34.771002Z","shell.execute_reply":"2022-05-25T19:34:35.844487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The raw audio file is time-seris data which indicates the amplitude of the sound over time (right figures). \n\nHowever, deep learning models rarely take this raw audio directly as input, the common practice is to convert the audio into a spectrogram (left figures). The spectrogram is a concise ‘snapshot’ of an audio wave and since it is an image, it is well suited to being input to CNN-based architectures developed for handling images. If you want to understand the audio spectrogram, this Audio Deep Learning [series](https://towardsdatascience.com/audio-deep-learning-made-simple-part-1-state-of-the-art-techniques-da1d3dff2504) is very useful. In this notebook, we use the spectrogram as input.","metadata":{}},{"cell_type":"markdown","source":"# Network\n\nThe implementation of the Variational Auto-Encoder (VAE) model.","metadata":{}},{"cell_type":"code","source":"class VAE(nn.Module):\n    \"\"\"Mostly borrowed from PyTorch example.\n    Thanks to https://github.com/pytorch/examples/blob/master/vae/main.py\n    \"\"\"\n\n    def __init__(self, device, x_dim=640, h_dim=400, z_dim=20):\n        super().__init__()\n        self.x_dim = x_dim\n\n        self.fc11 = nn.Linear(x_dim, h_dim)\n        self.fc12 = nn.Linear(h_dim, h_dim)\n        self.fc21 = nn.Linear(h_dim, z_dim)\n        self.fc22 = nn.Linear(h_dim, z_dim)\n        self.fc31 = nn.Linear(z_dim, h_dim)\n        self.fc32 = nn.Linear(h_dim, h_dim)\n        self.fc4 = nn.Linear(h_dim, x_dim)\n\n    def encode(self, x):\n        x = F.relu(self.fc11(x))\n        x = F.relu(self.fc12(x))\n        return self.fc21(x), self.fc22(x)\n\n    def reparameterize(self, mu, logvar):\n        std = torch.exp(0.5*logvar)\n        eps = torch.randn_like(std)\n        return mu + eps*std\n\n    def decode(self, z):\n        h = F.relu(self.fc31(z))\n        h = F.relu(self.fc32(h))\n        return torch.sigmoid(self.fc4(h))\n\n    def forward_all(self, x):\n        mu, logvar = self.encode(x.view(-1, self.x_dim))\n        z = self.reparameterize(mu, logvar)\n        return self.decode(z), z, mu, logvar\n\n    def forward(self, x):\n        yhat, _, _, _ = self.forward_all(x)\n        return yhat\n\n\ndef VAE_loss_function(recon_x, x, mu, logvar, reconst_loss='mse', a_RECONST=1., a_KLD=1., x_dim=640):\n    \"\"\"Loss function for VAE which consists of reconstruction and KL divergence losses.\n    Thanks to https://github.com/pytorch/examples/blob/master/vae/main.py\n\n    You can also balance weights for each loss, just to see what if KLD loss is stronger, etc.\n\n    Args:\n        reconst_loss: Reconstruction loss calculation: 'mse' or 'bce'\n        a_RECONST: Weight for reconstruction loss.\n        a_KLD: Weight for KLD loss.\n    \"\"\"\n\n    func = (F.mse_loss if reconst_loss == 'mse'\n            else F.binary_cross_entropy if reconst_loss == 'bce'\n            else 'Unknown reconst_loss')\n    RECONST = func(recon_x, x.view(-1, x_dim), reduction='sum')\n\n    # see Appendix B from VAE paper:\n    # Kingma and Welling. Auto-Encoding Variational Bayes. ICLR, 2014\n    # https://arxiv.org/abs/1312.6114\n    # 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)\n    KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())\n\n    return RECONST*a_RECONST + KLD*a_KLD\n","metadata":{"execution":{"iopub.status.busy":"2022-05-25T19:34:43.855819Z","iopub.execute_input":"2022-05-25T19:34:43.85609Z","iopub.status.idle":"2022-05-25T19:34:43.948515Z","shell.execute_reply.started":"2022-05-25T19:34:43.85606Z","shell.execute_reply":"2022-05-25T19:34:43.947705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parameters","metadata":{"execution":{"iopub.status.busy":"2022-05-02T08:38:36.395212Z","iopub.execute_input":"2022-05-02T08:38:36.395652Z","iopub.status.idle":"2022-05-02T08:38:36.422764Z","shell.execute_reply.started":"2022-05-02T08:38:36.395533Z","shell.execute_reply":"2022-05-02T08:38:36.422138Z"}}},{"cell_type":"code","source":"params = {\n# inout directory\n\"dev_directory\" : \"../input/eurecom-aml-2022-challenge-2/dev_data/dev_data\",\n\"eval_directory\" : \"../input/eurecom-aml-2022-challenge-2/eval_data/eval_data\",\n\"model_directory\": \"./model\",\n\"result_directory\": \"./result\",\n\"result_file\": \"result.csv\",\n\"max_fpr\" : 0.1,\n\"mode\" : True, # mode=True for development dataset, mode=False for evaluation dataset\n\n# preprocessing for mel-spectrogram\n\"feature\": {\n\"n_mels\": 128,\n\"frames\" : 5,\n\"n_fft\": 1024,\n\"hop_length\": 512,\n\"power\": 2.0},\n\n# training\n\"fit\":{\n\"lr\": 0.001,\n\"b1\": 0.9,\n\"b2\": 0.999,\n\"weight_decay\": 0.0,\n\"epochs\" : 100,\n\"batch_size\" : 1000,\n\"shuffle\" : True,\n\"validation_split\" : 0.1,\n\"verbose\" : 1},\n    \n# model architecture\n\"VAE\":{\n\"x_dim\": 640,\n\"h_dim\": 400,\n\"z_dim\": 20}\n    \n}\nparams = EasyDict(params)\n\n# create working directory\nensure_folder(params.model_directory)\n\n# test directories\ndirs = com.select_dirs(param=params, mode=params.mode)\n\n# fix random seeds\ndeterministic_everything(2020, pytorch=True)\n\n# PyTorch device\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2022-05-25T19:34:45.486156Z","iopub.execute_input":"2022-05-25T19:34:45.486725Z","iopub.status.idle":"2022-05-25T19:34:45.639123Z","shell.execute_reply.started":"2022-05-25T19:34:45.486684Z","shell.execute_reply":"2022-05-25T19:34:45.638224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"class Task2Dataset(torch.utils.data.Dataset):\n    \"\"\"PyTorch dataset class for task2. Caching to a file supported.\n\n    Args:\n        n_mels, frames, n_fft, hop_length, power, transform: Audio conversion settings.\n        normalize: Normalize data value range from [-90, 24] to [0, 1] for VAE, False by default.\n        cache_to: Cache filename or None by default, use this for your iterative development.\n    \"\"\"\n\n    def __init__(self, files, n_mels, frames, n_fft, hop_length, power, transform,\n                 normalize=False, cache_to=None):\n        self.transform = transform\n        self.files = files\n        self.n_mels, self.frames, self.n_fft = n_mels, frames, n_fft\n        self.hop_length, self.power = hop_length, power\n        # load cache or convert all the data for the first time\n        if cache_to is not None and Path(cache_to).exists():\n            com.logger.info(f'Loading cached {Path(cache_to).name}')\n            self.X = np.load(cache_to)\n        else:\n            self.X = com.list_to_vector_array(self.files,\n                             n_mels=self.n_mels,\n                             frames=self.frames,\n                             n_fft=self.n_fft,\n                             hop_length=self.hop_length,\n                             power=self.power)\n            if cache_to is not None:\n                np.save(cache_to, self.X)\n\n        if normalize:\n            self.X = normalize_0to1(self.X)\n\n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, index):\n        x = self.X[index]\n        x = self.transform(x)\n        return x, x\n\n\n# VAE Training class\nclass Task2VAELightning(pl.LightningModule):\n    \"\"\"Task2 PyTorch Lightning class, for training only.\"\"\"\n\n    def __init__(self, device, model, params, files, normalize=False):\n        super().__init__()\n        self.params = params\n        self.normalize = normalize\n        self.model = model\n        self.mseloss = torch.nn.MSELoss()\n        # split data files\n        if files is not None:\n            n_val = int(params.fit.validation_split * len(files))\n            self.val_files = random.sample(files, n_val)\n            self.train_files = [f for f in files if f not in self.val_files]\n\n    def forward(self, x):\n        return self.model(x)\n    \n    def training_step(self, batch, batch_nb):\n        x, y = batch\n        y_hat, z, mu, logvar = self.model.forward_all(x)\n        loss = VAE_loss_function(recon_x=y_hat, x=x, mu=mu, logvar=logvar,\n                                 reconst_loss='mse',\n                                 a_RECONST=1.,\n                                 a_KLD=.01)\n        tensorboard_logs = {'train_loss': loss}\n        return {'loss': loss, 'log': tensorboard_logs}\n\n    def validation_step(self, batch, batch_nb):\n        x, y = batch\n        y_hat = self.forward(x)\n        return {'val_loss': self.mseloss(y_hat, y)}\n\n    def validation_epoch_end(self, outputs):\n        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n        tensorboard_logs = {'val_loss': avg_loss}\n        return {'avg_val_loss': avg_loss, 'log': tensorboard_logs}\n\n    def configure_optimizers(self):\n        return torch.optim.Adam(self.parameters(), lr=self.params.fit.lr,\n                                betas=(self.params.fit.b1, self.params.fit.b2),\n                                weight_decay=self.params.fit.weight_decay)\n\n    def _get_dl(self, for_what):\n        files = self.train_files if for_what == 'train' else self.val_files\n        cache_file = f'{self.params.model_directory}/__cache_{str(files[0]).split(\"/\")[-3]}_{for_what}.npy'\n        ds = Task2Dataset(files,\n                          n_mels=self.params.feature.n_mels,\n                          frames=self.params.feature.frames,\n                          n_fft=self.params.feature.n_fft,\n                          hop_length=self.params.feature.hop_length,\n                          power=self.params.feature.power,\n                          transform=ToTensor1ch(device=device),\n                          normalize=self.normalize,\n                          cache_to=cache_file)\n        return torch.utils.data.DataLoader(ds, batch_size=self.params.fit.batch_size,\n                          shuffle=(self.params.fit.shuffle if for_what == 'train' else False))\n\n    def train_dataloader(self):\n        return self._get_dl('train')\n\n    def val_dataloader(self):\n        return self._get_dl('val')","metadata":{"execution":{"iopub.status.busy":"2022-05-25T19:34:47.380229Z","iopub.execute_input":"2022-05-25T19:34:47.380544Z","iopub.status.idle":"2022-05-25T19:34:47.473814Z","shell.execute_reply.started":"2022-05-25T19:34:47.380509Z","shell.execute_reply":"2022-05-25T19:34:47.472738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train models\n\ndef train():\n    for target_dir in dirs:\n        target = str(target_dir).split('/')[-1]\n        print(f'==== Start training [{target}] with {torch.cuda.device_count()} GPU(s). ====')\n\n        files = com.file_list_generator(target_dir)\n\n        model = VAE(device, x_dim=params.VAE.x_dim, h_dim=params.VAE.h_dim, z_dim=params.VAE.z_dim).to(device)\n        summary(device, model)\n        task2 = Task2VAELightning(device, model, params, files, normalize=True)\n        trainer = pl.Trainer(max_epochs=params.fit.epochs,\n                             gpus=torch.cuda.device_count())\n        trainer.fit(task2)\n\n        model_file = f'{params.model_directory}/model_{target}.pth'\n        torch.save(task2.model.state_dict(), model_file)\n        print(f'saved {model_file}.\\n')\n        \ntrain()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T19:34:47.718548Z","iopub.execute_input":"2022-05-25T19:34:47.719394Z","iopub.status.idle":"2022-05-25T20:39:06.906382Z","shell.execute_reply.started":"2022-05-25T19:34:47.719349Z","shell.execute_reply":"2022-05-25T20:39:06.905536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Testing on Development dataset","metadata":{}},{"cell_type":"code","source":"def test():\n    # make output result directory\n    os.makedirs(params.result_directory, exist_ok=True)\n\n    # load base directory\n    dirs = com.select_dirs(param=params, mode=params.mode)\n\n    # initialize lines in csv for AUC and pAUC\n    csv_lines = []\n\n    # PyTorch version specific...\n    to_tensor = pytorch_common.ToTensor1ch()\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n    # loop of the base directory\n    for idx, target_dir in enumerate(dirs):\n        print(\"\\n===========================\")\n        print(\"[{idx}/{total}] {dirname}\".format(dirname=target_dir, idx=idx+1, total=len(dirs)))\n        machine_type = os.path.split(target_dir)[1]\n\n        print(\"============== MODEL LOAD ==============\")\n        # set model path\n        model_file = \"{model}/model_{machine_type}.pth\".format(model=params.model_directory,\n                                                               machine_type=machine_type)\n\n        # load model file\n        if not os.path.exists(model_file):\n            com.logger.error(\"{} model not found \".format(machine_type))\n            sys.exit(-1)\n        com.logger.info(\"loading model: {}\".format(model_file))\n        model = VAE(device, x_dim=params.VAE.x_dim, h_dim=params.VAE.h_dim, z_dim=params.VAE.z_dim).to(device)\n        pytorch_common.load_weights(model, model_file)\n        pytorch_common.summary(device, model)\n        model.eval()\n\n        if params.mode:\n            # results by type\n            csv_lines.append([machine_type])\n            csv_lines.append([\"id\", \"AUC\", \"pAUC\"])\n            performance = []\n\n        machine_id_list = get_machine_id_list_for_test(target_dir)\n\n        for id_str in machine_id_list:\n            # load test file\n            test_files, y_true = test_file_list_generator(target_dir, id_str, mode=params.mode)\n\n            # setup anomaly score file path\n            anomaly_score_csv = \"{result}/anomaly_score_{machine_type}_{id_str}.csv\".format(\n                                                                                     result=params.result_directory,\n                                                                                     machine_type=machine_type,\n                                                                                     id_str=id_str)\n            anomaly_score_list = []\n\n            print(\"\\n============== BEGIN TEST FOR A MACHINE ID ==============\")\n            y_pred = [0. for k in test_files]\n            for file_idx, file_path in tqdm(enumerate(test_files), total=len(test_files)):\n                try:\n                    data = com.file_to_vector_array(file_path,\n                                                    n_mels=params.feature.n_mels,\n                                                    frames=params.feature.frames,\n                                                    n_fft=params.feature.n_fft,\n                                                    hop_length=params.feature.hop_length,\n                                                    power=params.feature.power)\n                    data = pytorch_common.normalize_0to1(data)\n                    with torch.no_grad():\n                        yhat = model(to_tensor(data)).cpu().detach().numpy().reshape(data.shape)\n                        errors = numpy.mean(numpy.square(data - yhat), axis=1)\n                    y_pred[file_idx] = numpy.mean(errors)\n                    anomaly_score_list.append([os.path.basename(file_path), y_pred[file_idx]])\n                except:\n                    com.logger.error(\"file broken!!: {}\".format(file_path))\n                    sys.exit(-1)\n\n            # save anomaly score\n            save_csv(save_file_path=anomaly_score_csv, save_data=anomaly_score_list)\n            com.logger.info(\"anomaly score result ->  {}\".format(anomaly_score_csv))\n\n            if params.mode:\n                # append AUC and pAUC to lists\n                auc = metrics.roc_auc_score(y_true, y_pred)\n                p_auc = metrics.roc_auc_score(y_true, y_pred, max_fpr=params.max_fpr)\n                csv_lines.append([id_str.split(\"_\", 1)[1], auc, p_auc])\n                performance.append([auc, p_auc])\n                com.logger.info(\"AUC : {}\".format(auc))\n                com.logger.info(\"pAUC : {}\".format(p_auc))\n\n            print(\"\\n============ END OF TEST FOR A MACHINE ID ============\")\n\n        if params.mode:\n            # calculate averages for AUCs and pAUCs\n            averaged_performance = numpy.mean(numpy.array(performance, dtype=float), axis=0)\n            csv_lines.append([\"Average\"] + list(averaged_performance))\n            csv_lines.append([])\n\n    if params.mode:\n        # output results\n        result_path = \"{result}/{file_name}\".format(result=params.result_directory, file_name=params.result_file)\n        com.logger.info(\"AUC and pAUC results -> {}\".format(result_path))\n        save_csv(save_file_path=result_path, save_data=csv_lines)\n\ntest()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T20:39:06.908237Z","iopub.execute_input":"2022-05-25T20:39:06.908781Z","iopub.status.idle":"2022-05-25T20:39:42.777768Z","shell.execute_reply.started":"2022-05-25T20:39:06.90874Z","shell.execute_reply":"2022-05-25T20:39:42.776856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def upto_6digits(cell):\n    if not cell[0].isdigit(): return cell\n    return f'{float(cell):.6f}'\n\nif params.mode:\n    with open('./result/result.csv') as f:\n        for l in f.readlines():\n            l = l.strip() #replace('\\n', '')\n            if ',' not in l:\n                print(l)\n                continue\n            ls = l.split(',')\n            print(f'{ls[0]}\\t\\t{upto_6digits(ls[1])}\\t\\t{upto_6digits(ls[2])}')","metadata":{"execution":{"iopub.status.busy":"2022-05-25T21:02:40.118875Z","iopub.execute_input":"2022-05-25T21:02:40.119383Z","iopub.status.idle":"2022-05-25T21:02:40.19166Z","shell.execute_reply.started":"2022-05-25T21:02:40.119344Z","shell.execute_reply":"2022-05-25T21:02:40.190943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Testing on the Evaluation dataset and generating submission file","metadata":{}},{"cell_type":"code","source":"# re-training on evaluation dataset\nparams.mode = False\n# train() # due to the memory restriction, restart the kernel and train on only evaluation dataset","metadata":{"execution":{"iopub.status.busy":"2022-05-25T21:02:55.880717Z","iopub.execute_input":"2022-05-25T21:02:55.88099Z","iopub.status.idle":"2022-05-25T21:02:55.949942Z","shell.execute_reply.started":"2022-05-25T21:02:55.88096Z","shell.execute_reply":"2022-05-25T21:02:55.948957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T21:02:56.577253Z","iopub.execute_input":"2022-05-25T21:02:56.57816Z","iopub.status.idle":"2022-05-25T21:03:13.936832Z","shell.execute_reply.started":"2022-05-25T21:02:56.578105Z","shell.execute_reply":"2022-05-25T21:03:13.936017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\ndf1 = pd.read_csv('./result/anomaly_score_slider_id_01.csv', delimiter=',', header=None)\ndf3 = pd.read_csv('./result/anomaly_score_slider_id_03.csv', delimiter=',', header=None)\ndf5 = pd.read_csv('./result/anomaly_score_slider_id_05.csv', delimiter=',', header=None)\n\ndfs = [df1, df3, df5]\neval_df = pd.concat(dfs)\neval_df = eval_df.rename(columns={0: \"file_name\", 1: \"anomaly_score\"})\n\neval_df.to_csv('./result/baseline_submission.csv',index = False)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-25T21:03:15.608209Z","iopub.execute_input":"2022-05-25T21:03:15.608753Z","iopub.status.idle":"2022-05-25T21:03:15.706935Z","shell.execute_reply.started":"2022-05-25T21:03:15.608715Z","shell.execute_reply":"2022-05-25T21:03:15.706076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T21:03:16.613089Z","iopub.execute_input":"2022-05-25T21:03:16.613559Z","iopub.status.idle":"2022-05-25T21:03:16.694532Z","shell.execute_reply.started":"2022-05-25T21:03:16.613515Z","shell.execute_reply":"2022-05-25T21:03:16.693498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}