{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#from google.colab import drive\n#drive.mount('/content/drive')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install torchsummary\n!pip install dl-cliche\n#!pip install pytorch-lightning","metadata":{"execution":{"iopub.status.busy":"2022-06-12T19:38:00.350739Z","iopub.execute_input":"2022-06-12T19:38:00.351229Z","iopub.status.idle":"2022-06-12T19:38:20.553052Z","shell.execute_reply.started":"2022-06-12T19:38:00.351128Z","shell.execute_reply":"2022-06-12T19:38:20.551932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!pip install librosa --upgrade","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from shutil import copyfile\n\n# copy our file into the working directory (make sure it has .py suffix)\ncopyfile(src = \"../input/dcase2020-task2-baseline/pytorch_common.py\", dst = \"../working/pytorch_common.py\")\ncopyfile(src = \"../input/dcase2020-task2-baseline/common.py\", dst = \"../working/common.py\")\n\nimport sys\n#sys.path.append(\"drive/MyDrive/Colab Notebooks\")\nimport common as com\nimport pytorch_common\nfrom pytorch_common import *","metadata":{"execution":{"iopub.status.busy":"2022-06-12T19:38:30.471217Z","iopub.execute_input":"2022-06-12T19:38:30.471902Z","iopub.status.idle":"2022-06-12T19:38:40.893252Z","shell.execute_reply.started":"2022-06-12T19:38:30.471859Z","shell.execute_reply":"2022-06-12T19:38:40.892361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport librosa\nimport glob\nimport csv\nimport re\nimport itertools\nimport random\nimport seaborn as sns\nfrom tqdm import tqdm\nfrom scipy import signal\nimport soundfile as sf\nimport librosa.display\nfrom sklearn import metrics\n\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nimport torchsummary\nimport pytorch_lightning as pl\n\nfrom dlcliche.utils import *\nfrom dlcliche.notebook import *\nfrom dlcliche.utils import (\n    sys, random, Path, np, plt, EasyDict,\n    ensure_folder, deterministic_everything,\n)","metadata":{"execution":{"iopub.status.busy":"2022-06-12T19:38:44.175960Z","iopub.execute_input":"2022-06-12T19:38:44.176704Z","iopub.status.idle":"2022-06-12T19:38:44.324113Z","shell.execute_reply.started":"2022-06-12T19:38:44.176665Z","shell.execute_reply":"2022-06-12T19:38:44.323394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed: int):    \n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    \n    #tf.random.set_seed(seed)\n    #os.environ['TF_DETERMINISTIC_OPS'] = '1'\n    #os.environ['TF_CUDNN_DETERMINISTIC'] = '1'\n    #tf.config.threading.set_inter_op_parallelism_threads(1)\n    #tf.config.threading.set_intra_op_parallelism_threads(1)\n\nseed = 42\nseed_everything(seed)","metadata":{"execution":{"iopub.status.busy":"2022-06-12T19:38:53.591281Z","iopub.execute_input":"2022-06-12T19:38:53.591972Z","iopub.status.idle":"2022-06-12T19:38:53.668327Z","shell.execute_reply.started":"2022-06-12T19:38:53.591927Z","shell.execute_reply":"2022-06-12T19:38:53.667400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Data Exploration and Preprocessing**","metadata":{}},{"cell_type":"code","source":"# development dataset\nDATA_ROOT = Path(\"../input/eurecom-aml-2022-challenge-2/dev_data/dev_data\")\n#DATA_ROOT = Path(\"drive/MyDrive/AML/dev_data/dev_data\")","metadata":{"execution":{"iopub.status.busy":"2022-06-12T19:39:11.108307Z","iopub.execute_input":"2022-06-12T19:39:11.109296Z","iopub.status.idle":"2022-06-12T19:39:11.182342Z","shell.execute_reply.started":"2022-06-12T19:39:11.109252Z","shell.execute_reply":"2022-06-12T19:39:11.181402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"types = [t.name for t in sorted(DATA_ROOT.glob('*')) if t.is_dir()]\nprint('Machine types:', types)\n\ndf = pd.DataFrame()\ndf['file'] = sorted(DATA_ROOT.glob('*/*/*.wav'))\ndf['type'] = df.file.map(lambda f: f.parent.parent.name)\ndf['split'] = df.file.map(lambda f: f.parent.name)\ndf['id'] = df.file.map(lambda s: str(s).split('/')[-1].split('_')[-2])\n\nagg = df.groupby(['id', 'split']).agg('count')\nfig = plt.figure(figsize=(12.0, 6.0))\ng = sns.barplot(x=\"id\", y=\"file\", hue=\"split\", data=agg.reset_index())\nplt.xlabel(\"machine id\")\nplt.ylabel(\"number of files\")\nplt.show()\nagg.transpose()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T19:39:13.667821Z","iopub.execute_input":"2022-06-12T19:39:13.668620Z","iopub.status.idle":"2022-06-12T19:39:14.425285Z","shell.execute_reply.started":"2022-06-12T19:39:13.668578Z","shell.execute_reply":"2022-06-12T19:39:14.424409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train=df[df.split=='train']\ndf_test=df[df.split=='test']","metadata":{"execution":{"iopub.status.busy":"2022-06-12T19:39:18.377905Z","iopub.execute_input":"2022-06-12T19:39:18.378625Z","iopub.status.idle":"2022-06-12T19:39:18.454436Z","shell.execute_reply.started":"2022-06-12T19:39:18.378584Z","shell.execute_reply":"2022-06-12T19:39:18.453375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg = df[['file', 'type', 'split']].groupby(['type', 'split']).agg('count')\nfig = plt.figure(figsize=(12.0, 6.0))\ng = sns.barplot(x=\"type\", y=\"file\", hue=\"split\", data=agg.reset_index())\nplt.ylabel(\"machine type\")\nplt.ylabel(\"number of files\")\nplt.show()\nagg.transpose()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T19:39:20.723808Z","iopub.execute_input":"2022-06-12T19:39:20.724278Z","iopub.status.idle":"2022-06-12T19:39:21.021690Z","shell.execute_reply.started":"2022-06-12T19:39:20.724238Z","shell.execute_reply":"2022-06-12T19:39:21.020904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# evaluation dataset\nDATA_ROOT = Path(\"../input/eurecom-aml-2022-challenge-2/eval_data/eval_data\")\n#DATA_ROOT = Path(\"drive/MyDrive/AML/eval_data/eval_data\")","metadata":{"execution":{"iopub.status.busy":"2022-06-12T19:39:26.162108Z","iopub.execute_input":"2022-06-12T19:39:26.162518Z","iopub.status.idle":"2022-06-12T19:39:26.235085Z","shell.execute_reply.started":"2022-06-12T19:39:26.162482Z","shell.execute_reply":"2022-06-12T19:39:26.234144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"types = [t.name for t in sorted(DATA_ROOT.glob('*')) if t.is_dir()]\nprint('Machine types:', types)\n\ndf_eval = pd.DataFrame()\ndf_eval['file'] = sorted(DATA_ROOT.glob('*/*/*.wav'))\ndf_eval['type'] = df_eval.file.map(lambda f: f.parent.parent.name)\ndf_eval['split'] = df_eval.file.map(lambda f: f.parent.name)\ndf_eval['id'] = df_eval.file.map(lambda s: str(s).split('/')[-1].split('_')[-2])\n\nagg = df_eval.groupby(['id', 'split']).agg('count')\nfig = plt.figure(figsize=(12.0, 6.0))\ng = sns.barplot(x=\"id\", y=\"file\", hue=\"split\", data=agg.reset_index())\nplt.xlabel(\"machine id\")\nplt.ylabel(\"number of files\")\nplt.show()\nagg.transpose()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T19:39:29.031909Z","iopub.execute_input":"2022-06-12T19:39:29.032338Z","iopub.status.idle":"2022-06-12T19:39:29.766927Z","shell.execute_reply.started":"2022-06-12T19:39:29.032301Z","shell.execute_reply":"2022-06-12T19:39:29.766118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"agg = df_eval[['file', 'type', 'split']].groupby(['type', 'split']).agg('count')\nfig = plt.figure(figsize=(12.0, 6.0))\ng = sns.barplot(x=\"type\", y=\"file\", hue=\"split\", data=agg.reset_index())\nplt.ylabel(\"machine type\")\nplt.ylabel(\"number of files\")\nplt.show()\nagg.transpose()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T19:39:33.435313Z","iopub.execute_input":"2022-06-12T19:39:33.435712Z","iopub.status.idle":"2022-06-12T19:39:33.686706Z","shell.execute_reply.started":"2022-06-12T19:39:33.435681Z","shell.execute_reply":"2022-06-12T19:39:33.685954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"normal_signal_file = df_train.file.iloc[1]\nnormal_signal, samplerate = sf.read(normal_signal_file)\nfig = plt.figure(figsize=(12, 6))\nlibrosa.display.waveshow(normal_signal, sr=samplerate, alpha=0.5, linewidth=0.5, label='Machine_Slider_Id_00_Normal_Sound')\nplt.title('Normal sound waveform')\nplt.legend();","metadata":{"execution":{"iopub.status.busy":"2022-06-12T17:57:28.675558Z","iopub.execute_input":"2022-06-12T17:57:28.676432Z","iopub.status.idle":"2022-06-12T17:57:29.421428Z","shell.execute_reply.started":"2022-06-12T17:57:28.676375Z","shell.execute_reply":"2022-06-12T17:57:29.419628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_fft=1024\nhop_length=512\nfig, ax = plt.subplots(figsize=(12, 6))\nS_normal = np.abs(librosa.stft(\n    normal_signal, n_fft=n_fft, hop_length=hop_length, center=False))\nlibrosa.display.specshow(\n    librosa.amplitude_to_db(S_normal, ref=np.max), n_fft=n_fft, \n    hop_length=hop_length, sr=samplerate, x_axis='time', y_axis='linear',\n    ax=ax, cmap='jet')\nax.set(title = 'Normal sound Spectrogram');","metadata":{"execution":{"iopub.status.busy":"2022-06-12T17:57:30.469915Z","iopub.execute_input":"2022-06-12T17:57:30.470922Z","iopub.status.idle":"2022-06-12T17:57:30.904014Z","shell.execute_reply.started":"2022-06-12T17:57:30.470866Z","shell.execute_reply":"2022-06-12T17:57:30.903282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anomaly_signal_file = df_test.file.iloc[1]\nanomaly_signal, samplerate = sf.read(anomaly_signal_file)\nfig = plt.figure(figsize=(12, 6))\nlibrosa.display.waveshow(anomaly_signal, sr=samplerate, alpha=0.5, linewidth=0.5, label='Machine_Slider_Id_00_Anomaly_Sound')\nplt.title('Anomaly sound waveform')\nplt.legend();","metadata":{"execution":{"iopub.status.busy":"2022-06-12T17:57:33.599425Z","iopub.execute_input":"2022-06-12T17:57:33.600156Z","iopub.status.idle":"2022-06-12T17:57:34.145073Z","shell.execute_reply.started":"2022-06-12T17:57:33.600117Z","shell.execute_reply":"2022-06-12T17:57:34.144191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_fft=1024\nhop_length=512\nfig, ax = plt.subplots(figsize=(12,6))\nS_anomaly = np.abs(librosa.stft(\n    anomaly_signal, n_fft=n_fft, hop_length=hop_length, center=False))\nlibrosa.display.specshow(\n    librosa.amplitude_to_db(S_anomaly, ref=np.max), n_fft=n_fft, \n    hop_length=hop_length, sr=samplerate, x_axis='time', y_axis='linear',\n    ax=ax, cmap='jet')\nax.set(title='Anomaly sound spectrogram');","metadata":{"execution":{"iopub.status.busy":"2022-06-12T17:57:35.180045Z","iopub.execute_input":"2022-06-12T17:57:35.180458Z","iopub.status.idle":"2022-06-12T17:57:35.621748Z","shell.execute_reply.started":"2022-06-12T17:57:35.180422Z","shell.execute_reply":"2022-06-12T17:57:35.620048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_log_mel_spectrogram(filename, n_mels=128,\n                        n_fft=1024,\n                        hop_length=512,\n                        power=2.0):\n    wav, sampling_rate = com.file_load(filename)\n    mel_spectrogram = librosa.feature.melspectrogram(y=wav,\n                                                     sr=sampling_rate,\n                                                     n_fft=n_fft,\n                                                     hop_length=hop_length,\n                                                     n_mels=n_mels,\n                                                     power=power)\n    log_mel_spectrogram = 20.0 / power * np.log10(mel_spectrogram + sys.float_info.epsilon)\n    return log_mel_spectrogram, wav\n\nfor t in types:\n    for split in ['train', 'test']:\n        type_df = df[df['type'] == t][df.split == split].reset_index()\n        R = 2\n        fig, ax = plt.subplots(R, 2, figsize = (15, 1.5*R))\n        print(f'=== Machine type [{t}], {split} set ===')\n        for i in range(R * 2):\n            file_index = i//2\n            file_path = Path(type_df.file[file_index])\n            mels, wav = get_log_mel_spectrogram(file_path)\n            if i == 0:\n                print(f\"Mel shape: {mels.shape}\")\n            if i%2==0:\n                ax[i//2, i%2].set_title(file_path.name)\n                ax[i//2, i%2].imshow(mels)\n                ax[i//2, i%2].axis('off')\n            else:\n                ax[i//2, i%2].set_title(file_path.name)\n                ax[i//2, i%2].plot(wav)\n                ax[i//2, i%2].get_xaxis().set_ticks([])\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T19:39:49.197475Z","iopub.execute_input":"2022-06-12T19:39:49.198110Z","iopub.status.idle":"2022-06-12T19:39:50.122799Z","shell.execute_reply.started":"2022-06-12T19:39:49.198066Z","shell.execute_reply":"2022-06-12T19:39:50.121041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **VAE Model**","metadata":{}},{"cell_type":"code","source":"class VAE(nn.Module):\n    \"\"\"Mostly borrowed from PyTorch example.\n    Thanks to https://github.com/pytorch/examples/blob/master/vae/main.py\n    \"\"\"\n\n    def __init__(self, device, x_dim=640, h_dim=400, z_dim=20):\n        super().__init__()\n        self.x_dim = x_dim\n\n        self.fc11 = nn.Linear(x_dim, h_dim)\n        self.fc12 = nn.Linear(h_dim, h_dim)\n        self.fc21 = nn.Linear(h_dim, z_dim)\n        self.fc22 = nn.Linear(h_dim, z_dim)\n        self.fc31 = nn.Linear(z_dim, h_dim)\n        self.fc32 = nn.Linear(h_dim, h_dim)\n        self.fc4 = nn.Linear(h_dim, x_dim)\n\n    def encode(self, x):\n        x = F.relu(self.fc11(x))\n        x = F.relu(self.fc12(x))\n        return self.fc21(x), self.fc22(x)\n\n    def reparameterize(self, mu, logvar):\n        std = torch.exp(0.5*logvar)\n        eps = torch.randn_like(std)\n        return mu + eps*std\n\n    def decode(self, z):\n        h = F.relu(self.fc31(z))\n        h = F.relu(self.fc32(h))\n        return torch.sigmoid(self.fc4(h))\n\n    def forward_all(self, x):\n        mu, logvar = self.encode(x.view(-1, self.x_dim))\n        z = self.reparameterize(mu, logvar)\n        return self.decode(z), z, mu, logvar\n\n    def forward(self, x):\n        yhat, _, _, _ = self.forward_all(x)\n        return yhat\n\n\ndef VAE_loss_function(recon_x, x, mu, logvar, reconst_loss='mse', a_RECONST=1., a_KLD=1., x_dim=640):\n    \"\"\"Loss function for VAE which consists of reconstruction and KL divergence losses.\n    Thanks to https://github.com/pytorch/examples/blob/master/vae/main.py\n\n    You can also balance weights for each loss, just to see what if KLD loss is stronger, etc.\n\n    Args:\n        reconst_loss: Reconstruction loss calculation: 'mse' or 'bce'\n        a_RECONST: Weight for reconstruction loss.\n        a_KLD: Weight for KLD loss.\n    \"\"\"\n\n    func = (F.mse_loss if reconst_loss == 'mse'\n            else F.binary_cross_entropy if reconst_loss == 'bce'\n            else 'Unknown reconst_loss')\n    RECONST = func(recon_x, x.view(-1, x_dim), reduction='sum')\n\n    # see Appendix B from VAE paper:\n    # Kingma and Welling. Auto-Encoding Variational Bayes. ICLR, 2014\n    # https://arxiv.org/abs/1312.6114\n    # 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)\n    KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())\n\n    return RECONST*a_RECONST + KLD*a_KLD","metadata":{"execution":{"iopub.status.busy":"2022-06-12T17:57:42.468268Z","iopub.execute_input":"2022-06-12T17:57:42.468684Z","iopub.status.idle":"2022-06-12T17:57:42.551418Z","shell.execute_reply.started":"2022-06-12T17:57:42.468651Z","shell.execute_reply":"2022-06-12T17:57:42.550485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = VAE(device = 'cuda')\nprint(model)","metadata":{"execution":{"iopub.status.busy":"2022-06-12T16:53:24.597558Z","iopub.execute_input":"2022-06-12T16:53:24.598101Z","iopub.status.idle":"2022-06-12T16:53:24.721858Z","shell.execute_reply.started":"2022-06-12T16:53:24.598057Z","shell.execute_reply":"2022-06-12T16:53:24.720932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n# inout directory\n\"dev_directory\" : \"../input/eurecom-aml-2022-challenge-2/dev_data/dev_data\",\n#\"dev_directory\" : \"drive/MyDrive/AML/dev_data/dev_data\",\n\"eval_directory\" : \"../input/eurecom-aml-2022-challenge-2/eval_data/eval_data\",\n#\"dev_directory\" : \"drive/MyDrive/AML/eval_data/eval_data\",\n\"model_directory\": \"./model\",\n#\"model_directory\": \"drive/MyDrive/AML/model\",\n\"result_directory\": \"./result\",\n#\"result_directory\": \"drive/MyDrive/AML/result\",\n\"result_file\": \"result.csv\",\n\"max_fpr\" : 0.1,\n\"mode\" : True, # mode=True for development dataset, mode=False for evaluation dataset\n\n# preprocessing for mel-spectrogram\n\"feature\": {\n\"n_mels\": 128,\n\"frames\" : 5,\n\"n_fft\": 1024,\n\"hop_length\": 512,\n\"power\": 2.0},\n\n# training\n\"fit\":{\n\"lr\": 0.001,\n\"b1\": 0.9,\n\"b2\": 0.999,\n\"weight_decay\": 0.0,\n\"epochs\" : 100,\n\"batch_size\" : 1000,\n\"shuffle\" : True,\n\"validation_split\" : 0.1,\n\"verbose\" : 1},\n    \n# model architecture\n\"VAE\":{\n\"x_dim\": 640,\n\"h_dim\": 400,\n\"z_dim\": 20}\n    \n}\nparams = EasyDict(params)\n\n# create working directory\nensure_folder(params.model_directory)\n\n# test directories\ndirs = com.select_dirs(param=params, mode=params.mode)\n\n# fix random seeds\ndeterministic_everything(2020, pytorch=True)\n\n# PyTorch device\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2022-06-12T20:15:28.359785Z","iopub.execute_input":"2022-06-12T20:15:28.360832Z","iopub.status.idle":"2022-06-12T20:15:28.447169Z","shell.execute_reply.started":"2022-06-12T20:15:28.360789Z","shell.execute_reply":"2022-06-12T20:15:28.446236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Task2Dataset(torch.utils.data.Dataset):\n    \"\"\"PyTorch dataset class for task2. Caching to a file supported.\n\n    Args:\n        n_mels, frames, n_fft, hop_length, power, transform: Audio conversion settings.\n        normalize: Normalize data value range from [-90, 24] to [0, 1] for VAE, False by default.\n        cache_to: Cache filename or None by default, use this for your iterative development.\n    \"\"\"\n\n    def __init__(self, files, n_mels, frames, n_fft, hop_length, power, transform,\n                 normalize=True, cache_to=None):\n        self.transform = transform\n        self.files = files\n        self.n_mels, self.frames, self.n_fft = n_mels, frames, n_fft\n        self.hop_length, self.power = hop_length, power\n        # load cache or convert all the data for the first time\n        if cache_to is not None and Path(cache_to).exists():\n            com.logger.info(f'Loading cached {Path(cache_to).name}')\n            self.X = np.load(cache_to)\n        else:\n            self.X = com.list_to_vector_array(self.files,\n                             n_mels=self.n_mels,\n                             frames=self.frames,\n                             n_fft=self.n_fft,\n                             hop_length=self.hop_length,\n                             power=self.power)\n            if cache_to is not None:\n                np.save(cache_to, self.X)\n\n        if normalize:\n            self.X = normalize_0to1(self.X)\n\n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, index):\n        x = self.X[index]\n        x = self.transform(x)\n        return x, x\n    \nclass Task2ImageDataset(torch.utils.data.Dataset):\n    \"\"\"Task 2 dataset to handle samples as 1 channel image.\n\n    Unlike other dataset, set `preprocessed_file` as preprocessed dataset filename.\n    For every output, this dataset class crop square image from this original data.\n\n    Number of total samples is `n_sampling` times number of _live data_.\n    _Live data_ is all the original data by default, and filtered by splitting functions.\n\n    - For training use, set `random=True`. This will yield randomly cropped square\n      image (64x64 for example) from the original sample (64x431 for 10s sample).\n    - For validation use, set `random=False`. Image will be cropped from fixed position.\n\n    Augmentation can be flexibly applied to either the output `x` only or `y` only, or both `x` and `y`.\n    `aug_x` and `aug_y` control this behavor.\n\n    Data split for training/validation can be done by using:\n        `get_index_by_pct()`: generate list of training index.\n        `train_split(train_index)`: set live data as original samples listed on `train_index`.\n        `val_split(train_index)`: set live data as original samples NOT listed on `train_index`.\n    Yields:\n        x: square image expected to be used as source.\n        y: square image expected to be used as reference for evaluating reconstructed image by training model.\n    \"\"\"\n\n    def __init__(self, preprocessed_file, n_sampling=10, transform=None, augment_tfm=None,\n                 normalize=True, random=True, aug_x=True, aug_y=False, debug=False):\n        self.n_sampling = n_sampling\n        self.transform, self.augment_tfm = transform, augment_tfm\n        self.random, self.aug_x, self.aug_y = random, aug_x, aug_y\n\n        self.X = np.load(preprocessed_file)\n        if normalize:\n            self.X = normalize_0to1(self.X)\n\n        if debug:\n            from dlcliche.utils import display\n            from dlcliche.math import np_describe\n            display(np_describe(self.X[0].cpu().np()))\n\n        self.orgX = self.X\n  \n    def get_index_by_pct(self, split_pct=0.1):\n        n = len(self.orgX)\n        return random.sample(range(n), k=(n - int(n * split_pct)))\n\n    def train_split(self, train_index):\n        self.train_index = train_index\n        self.X = self.orgX[train_index]\n    \n    def val_split(self, train_index):\n        n = len(self.orgX)\n        self.val_index = [i for i in range(n) if i not in train_index]\n        self.X = self.orgX[self.val_index]\n\n    def __len__(self):\n        return len(self.X) * self.n_sampling\n\n    def __getitem__(self, index):\n        file_index = index // self.n_sampling\n        part_index = index % self.n_sampling\n        x = self.X[file_index]\n        dim, length = x.shape\n\n        # crop square part of sample\n        if self.random:\n            # random crop\n            start = random.randint(0, length - dim)\n        else:\n            # crop with fixed position\n            start = (length // self.n_sampling) * part_index\n        start = min(start, length - dim)\n        x = x[:, start:start+dim]\n\n        # augmentation transform\n        y = x\n        if self.augment_tfm is not None:\n            tfm_x = self.augment_tfm(x)\n            if self.aug_x: x = tfm_x\n            if self.aug_y: y = tfm_x\n\n        # transform (convert to tensor here)\n        if self.transform is not None:\n            x = self.transform(x)\n            y = self.transform(y)\n        return x, y\n\n\n# VAE Training class\nclass Task2VAELightning(pl.LightningModule):\n    \"\"\"Task2 PyTorch Lightning class, for training only.\"\"\"\n\n    def __init__(self, device, model, params, files, normalize=True):\n        super().__init__()\n        self.params = params\n        self.normalize = normalize\n        self.model = model\n        self.mseloss = torch.nn.MSELoss()\n        # split data files\n        if files is not None:\n            n_val = int(params.fit.validation_split * len(files))\n            self.val_files = random.sample(files, n_val)\n            self.train_files = [f for f in files if f not in self.val_files]\n\n    def forward(self, x):\n        return self.model(x)\n    \n    def training_step(self, batch, batch_nb):\n        x, y = batch\n        y_hat, z, mu, logvar = self.model.forward_all(x)\n        loss = VAE_loss_function(recon_x=y_hat, x=x, mu=mu, logvar=logvar,\n                                 reconst_loss='mse',\n                                 a_RECONST=1.,\n                                 a_KLD=0.01)  \n        tensorboard_logs = {'train_loss': loss}\n        return {'loss': loss, 'log': tensorboard_logs}\n\n    def validation_step(self, batch, batch_nb):\n        x, y = batch\n        y_hat = self.forward(x)\n        return {'val_loss': self.mseloss(y_hat, y)}\n\n    def validation_epoch_end(self, outputs):\n        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n        tensorboard_logs = {'val_loss': avg_loss}\n        return {'avg_val_loss': avg_loss, 'log': tensorboard_logs}\n\n    def configure_optimizers(self):\n        return torch.optim.Adam(self.parameters(), lr=self.params.fit.lr,\n                                betas=(self.params.fit.b1, self.params.fit.b2),\n                                weight_decay=self.params.fit.weight_decay)\n\n    def _get_dl(self, for_what):\n        files = self.train_files if for_what == 'train' else self.val_files\n        cache_file = f'{self.params.model_directory}/__cache_{str(files[0]).split(\"/\")[-3]}_{for_what}.npy'\n        ds = Task2Dataset(files,\n                          n_mels=self.params.feature.n_mels,\n                          frames=self.params.feature.frames,\n                          n_fft=self.params.feature.n_fft,\n                          hop_length=self.params.feature.hop_length,\n                          power=self.params.feature.power,\n                          transform=ToTensor1ch(device=device),\n                          normalize=self.normalize,\n                          cache_to=cache_file)\n        return torch.utils.data.DataLoader(ds, batch_size=self.params.fit.batch_size,\n                          shuffle=(self.params.fit.shuffle if for_what == 'train' else False))\n\n    def train_dataloader(self):\n        return self._get_dl('train')\n\n    def val_dataloader(self):\n        return self._get_dl('val')\n    \n    \n# VAE Training class\nclass Task2CNNVAELightning(pl.LightningModule):\n    \"\"\"Task2 PyTorch Lightning class, for training only.\"\"\"\n\n    def __init__(self, device, model, params, files, normalize=True):\n        super().__init__()\n        self.params = params\n        self.normalize = normalize\n        self.model = model\n        self.mseloss = torch.nn.MSELoss()\n        # split data files\n        if files is not None:\n            n_val = int(params.fit.validation_split * len(files))\n            self.val_files = random.sample(files, n_val)\n            self.train_files = [f for f in files if f not in self.val_files]\n\n    def forward(self, x):\n        return self.model(x)\n    \n    def training_step(self, batch, batch_nb):\n        x, y = batch\n        y_hat = self.forward(x)\n        loss = self.mseloss(y_hat, y)\n        tensorboard_logs = {'train_loss': loss}\n        return {'loss': loss, 'log': tensorboard_logs}\n\n    def validation_step(self, batch, batch_nb):\n        x, y = batch\n        y_hat = self.forward(x)\n        return {'val_loss': self.mseloss(y_hat, y)}\n\n    def validation_epoch_end(self, outputs):\n        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n        tensorboard_logs = {'val_loss': avg_loss}\n        return {'avg_val_loss': avg_loss, 'log': tensorboard_logs}\n\n    def configure_optimizers(self):\n        return torch.optim.Adam(self.parameters(), lr=self.params.fit.lr,\n                                betas=(self.params.fit.b1, self.params.fit.b2),\n                                weight_decay=self.params.fit.weight_decay)\n\n    def _get_dl(self, for_what):\n        files = self.train_files if for_what == 'train' else self.val_files\n        cache_file = f'{self.params.model_directory}/__cache_{str(files[0]).split(\"/\")[-3]}_{for_what}.npy'\n        ds = Task2Dataset(files,\n                          n_mels=self.params.feature.n_mels,\n                          frames=self.params.feature.frames,\n                          n_fft=self.params.feature.n_fft,\n                          hop_length=self.params.feature.hop_length,\n                          power=self.params.feature.power,\n                          transform=ToTensor1ch(device=device),\n                          normalize=self.normalize,\n                          cache_to=cache_file)\n        return torch.utils.data.DataLoader(ds, batch_size=self.params.fit.batch_size,\n                          shuffle=(self.params.fit.shuffle if for_what == 'train' else False))\n\n    def train_dataloader(self):\n        return self._get_dl('train')\n\n    def val_dataloader(self):\n        return self._get_dl('val')\nclass Task2ImageLightning(Task2CNNVAELightning):\n    \"\"\"Task2 PyTorch Lightning class, for training only.\"\"\"\n\n    def __init__(self, device, model, params, preprocessed_file, normalize=True):\n        super().__init__(device, model, params, files=None, normalize=normalize)\n        self.params = params\n        self.normalize = normalize\n        self.model = model\n        self.mseloss = torch.nn.MSELoss()\n\n        to_tensor = ToTensor1ch(device=self.device, image=True)\n        self.trn_ds = Task2ImageDataset(preprocessed_file, transform=to_tensor,\n                                      normalize=normalize)\n        self.val_ds = Task2ImageDataset(preprocessed_file, transform=to_tensor,\n                                      normalize=normalize, random=False)\n        train_index = self.trn_ds.get_index_by_pct()\n        self.trn_ds.train_split(train_index)\n        self.val_ds.val_split(train_index)\n\n    def _get_dl(self, for_what):\n        ds = self.trn_ds if for_what == 'train' else self.val_ds\n        return torch.utils.data.DataLoader(ds, batch_size=self.params.fit.batch_size,\n                          shuffle=(self.params.fit.shuffle if for_what == 'train' else False))","metadata":{"execution":{"iopub.status.busy":"2022-06-12T17:57:57.030604Z","iopub.execute_input":"2022-06-12T17:57:57.030979Z","iopub.status.idle":"2022-06-12T17:57:57.148371Z","shell.execute_reply.started":"2022-06-12T17:57:57.030946Z","shell.execute_reply":"2022-06-12T17:57:57.147509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train():\n    for target_dir in dirs:\n        target = str(target_dir).split('/')[-1]\n        print(f'==== Start training [{target}] with {torch.cuda.device_count()} GPU(s). ====')\n\n        files = com.file_list_generator(target_dir)\n\n        model = VAE(device, x_dim=params.VAE.x_dim, h_dim=params.VAE.h_dim, z_dim=params.VAE.z_dim).to(device)\n        summary(device, model)\n        task2 = Task2VAELightning(device, model, params, files, normalize=True)\n        trainer = pl.Trainer(max_epochs=params.fit.epochs,\n                             gpus=torch.cuda.device_count())\n        trainer.fit(task2)\n\n        model_file = f'{params.model_directory}/model_{target}.pth'\n        torch.save(task2.model.state_dict(), model_file)\n        print(f'saved {model_file}.\\n')\n\ntrain()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T17:58:32.428521Z","iopub.execute_input":"2022-06-12T17:58:32.429166Z","iopub.status.idle":"2022-06-12T17:58:32.532174Z","shell.execute_reply.started":"2022-06-12T17:58:32.429119Z","shell.execute_reply":"2022-06-12T17:58:32.531268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test():\n    # make output result directory\n    os.makedirs(params.result_directory, exist_ok=True)\n\n    # load base directory\n    dirs = com.select_dirs(param=params, mode=params.mode)\n\n    # initialize lines in csv for AUC and pAUC\n    csv_lines = []\n\n    # PyTorch version specific...\n    to_tensor = pytorch_common.ToTensor1ch()\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n    # loop of the base directory\n    for idx, target_dir in enumerate(dirs):\n        print(\"\\n===========================\")\n        print(\"[{idx}/{total}] {dirname}\".format(dirname=target_dir, idx=idx+1, total=len(dirs)))\n        machine_type = os.path.split(target_dir)[1]\n\n        print(\"============== MODEL LOAD ==============\")\n        # set model path\n        model_file = \"{model}/model_{machine_type}.pth\".format(model=params.model_directory,\n                                                               machine_type=machine_type)\n\n        # load model file\n        if not os.path.exists(model_file):\n            com.logger.error(\"{} model not found \".format(machine_type))\n            sys.exit(-1)\n        com.logger.info(\"loading model: {}\".format(model_file))\n        model = VAE(device, x_dim=params.VAE.x_dim, h_dim=params.VAE.h_dim, z_dim=params.VAE.z_dim).to(device)\n        pytorch_common.load_weights(model, model_file)\n        pytorch_common.summary(device, model)\n        model.eval()\n\n        if params.mode:\n            # results by type\n            csv_lines.append([machine_type])\n            csv_lines.append([\"id\", \"AUC\", \"pAUC\"])\n            performance = []\n\n        machine_id_list = get_machine_id_list_for_test(target_dir)\n\n        for id_str in machine_id_list:\n            # load test file\n            test_files, y_true = test_file_list_generator(target_dir, id_str, mode=params.mode)\n\n            # setup anomaly score file path\n            anomaly_score_csv = \"{result}/anomaly_score_{machine_type}_{id_str}.csv\".format(\n                                                                                     result=params.result_directory,\n                                                                                     machine_type=machine_type,\n                                                                                     id_str=id_str)\n            anomaly_score_list = []\n\n            print(\"\\n============== BEGIN TEST FOR A MACHINE ID ==============\")\n            y_pred = [0. for k in test_files]\n            for file_idx, file_path in tqdm(enumerate(test_files), total=len(test_files)):\n                try:\n                    data = com.file_to_vector_array(file_path,\n                                                    n_mels=params.feature.n_mels,\n                                                    frames=params.feature.frames,\n                                                    n_fft=params.feature.n_fft,\n                                                    hop_length=params.feature.hop_length,\n                                                    power=params.feature.power)\n                    data = pytorch_common.normalize_0to1(data)\n                    with torch.no_grad():\n                        yhat = model(to_tensor(data)).cpu().detach().numpy().reshape(data.shape)\n                        errors = np.mean(np.square(data - yhat), axis=1)\n                    y_pred[file_idx] = np.mean(errors)\n                    anomaly_score_list.append([os.path.basename(file_path), y_pred[file_idx]])\n                except:\n                    com.logger.error(\"file broken!!: {}\".format(file_path))\n                    sys.exit(-1)\n\n            # save anomaly score\n            save_csv(save_file_path=anomaly_score_csv, save_data=anomaly_score_list)\n            com.logger.info(\"anomaly score result ->  {}\".format(anomaly_score_csv))\n\n            if params.mode:\n                # append AUC and pAUC to lists\n                auc = metrics.roc_auc_score(y_true, y_pred)\n                p_auc = metrics.roc_auc_score(y_true, y_pred, max_fpr=params.max_fpr)\n                csv_lines.append([id_str.split(\"_\", 1)[1], auc, p_auc])\n                performance.append([auc, p_auc])\n                com.logger.info(\"AUC : {}\".format(auc))\n                com.logger.info(\"pAUC : {}\".format(p_auc))\n\n            print(\"\\n============ END OF TEST FOR A MACHINE ID ============\")\n\n        if params.mode:\n            # calculate averages for AUCs and pAUCs\n            averaged_performance = np.mean(np.array(performance, dtype=float), axis=0)\n            csv_lines.append([\"Average\"] + list(averaged_performance))\n            csv_lines.append([])\n\n    if params.mode:\n        # output results\n        result_path = \"{result}/{file_name}\".format(result=params.result_directory, file_name=params.result_file)\n        com.logger.info(\"AUC and pAUC results -> {}\".format(result_path))\n        save_csv(save_file_path=result_path, save_data=csv_lines)\n\ntest()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T17:58:49.877894Z","iopub.execute_input":"2022-06-12T17:58:49.878279Z","iopub.status.idle":"2022-06-12T17:58:49.968404Z","shell.execute_reply.started":"2022-06-12T17:58:49.878247Z","shell.execute_reply":"2022-06-12T17:58:49.967364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def upto_6digits(cell):\n    if not cell[0].isdigit(): return cell\n    return f'{float(cell):.6f}'\n\nif params.mode:\n    with open('./result/result.csv') as f:\n        for l in f.readlines():\n            l = l.strip() #replace('\\n', '')\n            if ',' not in l:\n                print(l)\n                continue\n            ls = l.split(',')\n            print(f'{ls[0]}\\t\\t{upto_6digits(ls[1])}\\t\\t{upto_6digits(ls[2])}')","metadata":{"execution":{"iopub.status.busy":"2022-06-12T17:55:17.348624Z","iopub.execute_input":"2022-06-12T17:55:17.348982Z","iopub.status.idle":"2022-06-12T17:55:17.423037Z","shell.execute_reply.started":"2022-06-12T17:55:17.348951Z","shell.execute_reply":"2022-06-12T17:55:17.421903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params.mode = False\ntrain()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T17:58:57.616456Z","iopub.execute_input":"2022-06-12T17:58:57.616857Z","iopub.status.idle":"2022-06-12T18:52:56.130796Z","shell.execute_reply.started":"2022-06-12T17:58:57.616825Z","shell.execute_reply":"2022-06-12T18:52:56.129877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params.mode = False\ntest()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T18:52:56.134262Z","iopub.execute_input":"2022-06-12T18:52:56.134852Z","iopub.status.idle":"2022-06-12T18:53:20.571460Z","shell.execute_reply.started":"2022-06-12T18:52:56.134818Z","shell.execute_reply":"2022-06-12T18:53:20.570585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = pd.read_csv('./result/anomaly_score_slider_id_01.csv', delimiter=',', header=None)\ndf3 = pd.read_csv('./result/anomaly_score_slider_id_03.csv', delimiter=',', header=None)\ndf5 = pd.read_csv('./result/anomaly_score_slider_id_05.csv', delimiter=',', header=None)\n#df1 = pd.read_csv('drive/MyDrive/AML/result/anomaly_score_slider_id_01.csv', delimiter=',', header = None)\n#df3 = pd.read_csv('drive/MyDrive/AML/result/anomaly_score_slider_id_03.csv', delimiter=',', header = None)\n#df5 = pd.read_csv('drive/MyDrive/AML/result/anomaly_score_slider_id_05.csv', delimiter=',', header = None)\n\ndfs = [df1, df3, df5]\neval_df = pd.concat(dfs)\neval_df = eval_df.rename(columns={0: \"file_name\", 1: \"anomaly_score\"})\n\neval_df.to_csv('/kaggle/working/submissionVAE.csv',index = False)\n#eval_df.to_csv('drive/MyDrive/AML/result/VAE.csv',index = False)","metadata":{"execution":{"iopub.status.busy":"2022-06-12T18:56:01.521998Z","iopub.execute_input":"2022-06-12T18:56:01.522440Z","iopub.status.idle":"2022-06-12T18:56:01.610084Z","shell.execute_reply.started":"2022-06-12T18:56:01.522398Z","shell.execute_reply":"2022-06-12T18:56:01.609117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T18:53:20.724873Z","iopub.execute_input":"2022-06-12T18:53:20.725513Z","iopub.status.idle":"2022-06-12T18:53:20.803494Z","shell.execute_reply.started":"2022-06-12T18:53:20.725474Z","shell.execute_reply":"2022-06-12T18:53:20.802616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **CNNAE Model**","metadata":{}},{"cell_type":"code","source":"#os.makedirs('drive/MyDrive/dev_data')\nos.makedirs('dev_data')\n#os.makedirs('drive/MyDrive/eval_data')\nos.makedirs('eval_data')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#SAVE_TO = Path(\"drive/MyDrive/dev_data\")\nSAVE_TO = Path(\"/kaggle/working/dev_data\")","metadata":{"execution":{"iopub.status.busy":"2022-06-12T19:31:23.648327Z","iopub.execute_input":"2022-06-12T19:31:23.649109Z","iopub.status.idle":"2022-06-12T19:31:23.721982Z","shell.execute_reply.started":"2022-06-12T19:31:23.649070Z","shell.execute_reply":"2022-06-12T19:31:23.721176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for t in types:\n    for split in ['train', 'test']:\n        type_df = df[df['type'] == t][df.split == split].reset_index()\n        # convert first file, we want to know the shape\n        mels, _ = get_log_mel_spectrogram(type_df.file[0])\n        # create big bucket to keep all the data\n        all_mels = np.zeros((len(type_df), mels.shape[0], mels.shape[1]))\n        filename = f'dc2020t2l1-{t}-{split}.npy'\n        # convert all files\n        print(f'{filename}: making {len(type_df)} log mel spectrogram data as shape {all_mels.shape}')\n        all_mels[0, :, :] = mels\n        print(type_df.file[0])\n        for i, f in enumerate(type_df.file[1:]):\n            all_mels[i, :, :], _ = get_log_mel_spectrogram(f)\n        np.save(SAVE_TO/filename, all_mels)","metadata":{"execution":{"iopub.status.busy":"2022-06-12T18:57:40.929942Z","iopub.execute_input":"2022-06-12T18:57:40.930536Z","iopub.status.idle":"2022-06-12T18:59:06.094803Z","shell.execute_reply.started":"2022-06-12T18:57:40.930498Z","shell.execute_reply":"2022-06-12T18:59:06.093844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#SAVE_TO = Path(\"drive/MyDrive/eval_data\")\nSAVE_TO = Path(\"/kaggle/working/eval_data\")","metadata":{"execution":{"iopub.status.busy":"2022-06-12T19:31:28.741832Z","iopub.execute_input":"2022-06-12T19:31:28.742469Z","iopub.status.idle":"2022-06-12T19:31:28.814155Z","shell.execute_reply.started":"2022-06-12T19:31:28.742431Z","shell.execute_reply":"2022-06-12T19:31:28.813417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for t in types:\n    for split in ['train', 'test']:\n        type_df = df_eval[df_eval['type'] == t][df_eval.split == split].reset_index()\n        # convert first file, we want to know the shape\n        mels, _ = get_log_mel_spectrogram(type_df.file[0])\n        # create big bucket to keep all the data\n        all_mels = np.zeros((len(type_df), mels.shape[0], mels.shape[1]))\n        filename = f'eval-dc2020t2l1-{t}-{split}.npy'\n        # convert all files\n        print(f'{filename}: making {len(type_df)} log mel spectrogram data as shape {all_mels.shape}')\n        all_mels[0, :, :] = mels\n        print(type_df.file[0])\n        for i, f in enumerate(type_df.file[1:]):\n            all_mels[i, :, :], _ = get_log_mel_spectrogram(f)\n        np.save(SAVE_TO/filename, all_mels)","metadata":{"execution":{"iopub.status.busy":"2022-06-12T18:59:40.528006Z","iopub.execute_input":"2022-06-12T18:59:40.528520Z","iopub.status.idle":"2022-06-12T19:00:52.698614Z","shell.execute_reply.started":"2022-06-12T18:59:40.528474Z","shell.execute_reply":"2022-06-12T19:00:52.697659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CNNAE(nn.Module):\n    def  __init__(self, z_dim=40):\n        super().__init__()\n\n        # define the network\n        # encoder\n        self.conv1 = nn.Sequential(nn.ZeroPad2d((1,2,1,2)),\n                              nn.Conv2d(1, 32, kernel_size=5, stride=2),\n                              nn.BatchNorm2d(32),\n                              nn.ReLU())\n        self.conv2 = nn.Sequential(nn.ZeroPad2d((1,2,1,2)),\n                              nn.Conv2d(32, 64, kernel_size=5, stride=2),\n                              nn.BatchNorm2d(64), nn.Dropout(0.2),\n                              nn.ReLU())\n        self.conv3 = nn.Sequential(nn.Conv2d(64, 128, kernel_size=3, stride=2, padding=0),\n                              nn.BatchNorm2d(128), nn.Dropout(0.2),\n                              nn.ReLU())\n        self.conv4 = nn.Sequential(nn.Conv2d(128, 256, kernel_size=3, stride=2, padding=0),\n                              nn.BatchNorm2d(256), nn.Dropout(0.2),\n                              nn.ReLU())\n        self.fc1 = nn.Conv2d(256, z_dim, kernel_size=3)\n\n        # decoder\n        self.fc2 = nn.Sequential(nn.ConvTranspose2d(z_dim, 256, kernel_size=3),\n                            nn.BatchNorm2d(256), nn.Dropout(0.2),\n                            nn.ReLU())\n        self.conv4d = nn.Sequential(nn.ConvTranspose2d(256, 128, kernel_size=3, stride=2, padding=0),\n                               nn.BatchNorm2d(128), nn.Dropout(0.2),\n                               nn.ReLU())\n        self.conv3d = nn.Sequential(nn.ConvTranspose2d(128, 64, kernel_size=3, stride=2, padding=0),\n                               nn.BatchNorm2d(64), nn.Dropout(0.2),\n                               nn.ReLU())\n        self.conv2d = nn.Sequential(nn.ConvTranspose2d(64, 32, kernel_size=5, stride=2),\n                               nn.ReLU())\n        self.conv1d = nn.ConvTranspose2d(32, 1, kernel_size=5, stride=2)\n\n    def forward(self, x):\n        #volume_size = int_shape(x)\n        encoded = self.fc1(self.conv4(self.conv3(self.conv2(self.conv1(x)))))\n\n        decoded = self.fc2(encoded)\n        decoded = self.conv4d(decoded)\n        decoded = self.conv3d(decoded)\n        decoded = self.conv2d(decoded)[:,:,1:-1,1:-1]\n        decoded = self.conv1d(decoded)[:,:,0:-1,0:-1]\n        decoded = nn.Sigmoid()(decoded)\n\n        return decoded","metadata":{"execution":{"iopub.status.busy":"2022-06-12T19:40:46.858728Z","iopub.execute_input":"2022-06-12T19:40:46.859647Z","iopub.status.idle":"2022-06-12T19:40:46.944235Z","shell.execute_reply.started":"2022-06-12T19:40:46.859607Z","shell.execute_reply":"2022-06-12T19:40:46.943204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CNNAE()\nprint(model)","metadata":{"execution":{"iopub.status.busy":"2022-06-12T19:31:43.819825Z","iopub.execute_input":"2022-06-12T19:31:43.820223Z","iopub.status.idle":"2022-06-12T19:31:43.908916Z","shell.execute_reply.started":"2022-06-12T19:31:43.820189Z","shell.execute_reply":"2022-06-12T19:31:43.908072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params_cnnae = {\n# inout directory\n\"dev_directory\" : \"/kaggle/working/dev_data\",\n#\"dev_directory\" : \"drive/MyDrive/dev_data\",\n\"eval_directory\" : \"/kaggle/working/eval_data\",\n#\"eval_directory\" : \"drive/MyDrive/eval_data\",\n\"model_directory\": \"./model\",\n#\"model_directory\": \"drive/MyDrive/AML/model2\",\n\"result_directory\": \"./result\",\n#\"result_directory\": \"drive/MyDrive/AML/result2\",\n\"result_file\": \"result.csv\",\n\"max_fpr\" : 0.1,\n\"mode\" : True, # mode=True for development dataset, mode=False for evaluation dataset\n\n# preprocessing for mel-spectrogram\n\"feature\": {\n\"n_mels\": 128, #64\n\"frames\" : 5,\n\"n_fft\": 1024,\n\"hop_length\": 512,\n\"power\": 2.0}, #2.0\n\n# training\n\"fit\":{\n\"lr\": 0.0007,\n\"b1\": 0.9,\n\"b2\": 0.99,\n\"weight_decay\": 0.0,\n\"epochs\" : 100,\n\"batch_size\" : 64, #256\n\"shuffle\" : True,\n\"validation_split\" : 0.1,\n\"verbose\" : 1}\n    \n}\nparams_cnnae = EasyDict(params_cnnae)\n\n# create working directory\nensure_folder(params_cnnae.model_directory)\n\n\ntypes = ['slider']\n\ndata_files = sorted(Path(params_cnnae.dev_directory).glob('dc2020t2l1*.npy'))\ndata = {t:[f for f in data_files if t in str(f)] for t in types}\n\n# fix random seeds\ndeterministic_everything(42, pytorch=True)\n\n# PyTorch device\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\ndata, types","metadata":{"execution":{"iopub.status.busy":"2022-06-12T19:40:57.510186Z","iopub.execute_input":"2022-06-12T19:40:57.510749Z","iopub.status.idle":"2022-06-12T19:40:57.664911Z","shell.execute_reply.started":"2022-06-12T19:40:57.510700Z","shell.execute_reply":"2022-06-12T19:40:57.664064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Task2Dataset(torch.utils.data.Dataset):\n    \"\"\"PyTorch dataset class for task2. Caching to a file supported.\n\n    Args:\n        n_mels, frames, n_fft, hop_length, power, transform: Audio conversion settings.\n        normalize: Normalize data value range from [-90, 24] to [0, 1] for VAE, False by default.\n        cache_to: Cache filename or None by default, use this for your iterative development.\n    \"\"\"\n\n    def __init__(self, files, n_mels, frames, n_fft, hop_length, power, transform,\n                 normalize=True, cache_to=None):\n        self.transform = transform\n        self.files = files\n        self.n_mels, self.frames, self.n_fft = n_mels, frames, n_fft\n        self.hop_length, self.power = hop_length, power\n        # load cache or convert all the data for the first time\n        if cache_to is not None and Path(cache_to).exists():\n            com.logger.info(f'Loading cached {Path(cache_to).name}')\n            self.X = np.load(cache_to)\n        else:\n            self.X = com.list_to_vector_array(self.files,\n                             n_mels=self.n_mels,\n                             frames=self.frames,\n                             n_fft=self.n_fft,\n                             hop_length=self.hop_length,\n                             power=self.power)\n            if cache_to is not None:\n                np.save(cache_to, self.X)\n\n        if normalize:\n            self.X = normalize_0to1(self.X)\n\n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, index):\n        x = self.X[index]\n        x = self.transform(x)\n        return x, x\n    \nclass Task2ImageDataset(torch.utils.data.Dataset):\n    \"\"\"Task 2 dataset to handle samples as 1 channel image.\n\n    Unlike other dataset, set `preprocessed_file` as preprocessed dataset filename.\n    For every output, this dataset class crop square image from this original data.\n\n    Number of total samples is `n_sampling` times number of _live data_.\n    _Live data_ is all the original data by default, and filtered by splitting functions.\n\n    - For training use, set `random=True`. This will yield randomly cropped square\n      image (64x64 for example) from the original sample (64x431 for 10s sample).\n    - For validation use, set `random=False`. Image will be cropped from fixed position.\n\n    Augmentation can be flexibly applied to either the output `x` only or `y` only, or both `x` and `y`.\n    `aug_x` and `aug_y` control this behavor.\n\n    Data split for training/validation can be done by using:\n        `get_index_by_pct()`: generate list of training index.\n        `train_split(train_index)`: set live data as original samples listed on `train_index`.\n        `val_split(train_index)`: set live data as original samples NOT listed on `train_index`.\n    Yields:\n        x: square image expected to be used as source.\n        y: square image expected to be used as reference for evaluating reconstructed image by training model.\n    \"\"\"\n\n    def __init__(self, preprocessed_file, n_sampling=10, transform=None, augment_tfm=None,\n                 normalize=True, random=True, aug_x=True, aug_y=False, debug=False):\n        self.n_sampling = n_sampling\n        self.transform, self.augment_tfm = transform, augment_tfm\n        self.random, self.aug_x, self.aug_y = random, aug_x, aug_y\n\n        self.X = np.load(preprocessed_file)\n        if normalize:\n            self.X = normalize_0to1(self.X)\n\n        if debug:\n            from dlcliche.utils import display\n            from dlcliche.math import np_describe\n            display(np_describe(self.X[0].cpu().np()))\n\n        self.orgX = self.X\n  \n    def get_index_by_pct(self, split_pct=0.1):\n        n = len(self.orgX)\n        return random.sample(range(n), k=(n - int(n * split_pct)))\n\n    def train_split(self, train_index):\n        self.train_index = train_index\n        self.X = self.orgX[train_index]\n    \n    def val_split(self, train_index):\n        n = len(self.orgX)\n        self.val_index = [i for i in range(n) if i not in train_index]\n        self.X = self.orgX[self.val_index]\n\n    def __len__(self):\n        return len(self.X) * self.n_sampling\n\n    def __getitem__(self, index):\n        file_index = index // self.n_sampling\n        part_index = index % self.n_sampling\n        x = self.X[file_index]\n        dim, length = x.shape\n\n        # crop square part of sample\n        if self.random:\n            # random crop\n            start = random.randint(0, length - dim)\n        else:\n            # crop with fixed position\n            start = (length // self.n_sampling) * part_index\n        start = min(start, length - dim)\n        x = x[:, start:start+dim]\n\n        # augmentation transform\n        y = x\n        if self.augment_tfm is not None:\n            tfm_x = self.augment_tfm(x)\n            if self.aug_x: x = tfm_x\n            if self.aug_y: y = tfm_x\n\n        # transform (convert to tensor here)\n        if self.transform is not None:\n            x = self.transform(x)\n            y = self.transform(y)\n        return x, y\n\n\n# VAE Training class\nclass Task2VAELightning(pl.LightningModule):\n    \"\"\"Task2 PyTorch Lightning class, for training only.\"\"\"\n\n    def __init__(self, device, model, params, files, normalize=True):\n        super().__init__()\n        self.params = params\n        self.normalize = normalize\n        self.model = model\n        self.mseloss = torch.nn.MSELoss()\n        # split data files\n        if files is not None:\n            n_val = int(params.fit.validation_split * len(files))\n            self.val_files = random.sample(files, n_val)\n            self.train_files = [f for f in files if f not in self.val_files]\n\n    def forward(self, x):\n        return self.model(x)\n    \n    def training_step(self, batch, batch_nb):\n        x, y = batch\n        y_hat, z, mu, logvar = self.model.forward_all(x)\n        loss = VAE_loss_function(recon_x=y_hat, x=x, mu=mu, logvar=logvar,\n                                 reconst_loss='mse',\n                                 a_RECONST=1.,\n                                 a_KLD=0.01)  \n        tensorboard_logs = {'train_loss': loss}\n        return {'loss': loss, 'log': tensorboard_logs}\n\n    def validation_step(self, batch, batch_nb):\n        x, y = batch\n        y_hat = self.forward(x)\n        return {'val_loss': self.mseloss(y_hat, y)}\n\n    def validation_epoch_end(self, outputs):\n        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n        tensorboard_logs = {'val_loss': avg_loss}\n        return {'avg_val_loss': avg_loss, 'log': tensorboard_logs}\n\n    def configure_optimizers(self):\n        return torch.optim.Adam(self.parameters(), lr=self.params.fit.lr,\n                                betas=(self.params.fit.b1, self.params.fit.b2),\n                                weight_decay=self.params.fit.weight_decay)\n\n    def _get_dl(self, for_what):\n        files = self.train_files if for_what == 'train' else self.val_files\n        cache_file = f'{self.params.model_directory}/__cache_{str(files[0]).split(\"/\")[-3]}_{for_what}.npy'\n        ds = Task2Dataset(files,\n                          n_mels=self.params.feature.n_mels,\n                          frames=self.params.feature.frames,\n                          n_fft=self.params.feature.n_fft,\n                          hop_length=self.params.feature.hop_length,\n                          power=self.params.feature.power,\n                          transform=ToTensor1ch(device=device),\n                          normalize=self.normalize,\n                          cache_to=cache_file)\n        return torch.utils.data.DataLoader(ds, batch_size=self.params.fit.batch_size,\n                          shuffle=(self.params.fit.shuffle if for_what == 'train' else False))\n\n    def train_dataloader(self):\n        return self._get_dl('train')\n\n    def val_dataloader(self):\n        return self._get_dl('val')\n    \n    \n# VAE Training class\nclass Task2CNNVAELightning(pl.LightningModule):\n    \"\"\"Task2 PyTorch Lightning class, for training only.\"\"\"\n\n    def __init__(self, device, model, params, files, normalize=True):\n        super().__init__()\n        self.params = params\n        self.normalize = normalize\n        self.model = model\n        self.mseloss = torch.nn.MSELoss()\n        # split data files\n        if files is not None:\n            n_val = int(params.fit.validation_split * len(files))\n            self.val_files = random.sample(files, n_val)\n            self.train_files = [f for f in files if f not in self.val_files]\n\n    def forward(self, x):\n        return self.model(x)\n    \n    def training_step(self, batch, batch_nb):\n        x, y = batch\n        y_hat = self.forward(x)\n        loss = self.mseloss(y_hat, y)\n        tensorboard_logs = {'train_loss': loss}\n        return {'loss': loss, 'log': tensorboard_logs}\n\n    def validation_step(self, batch, batch_nb):\n        x, y = batch\n        y_hat = self.forward(x)\n        return {'val_loss': self.mseloss(y_hat, y)}\n\n    def validation_epoch_end(self, outputs):\n        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n        tensorboard_logs = {'val_loss': avg_loss}\n        return {'avg_val_loss': avg_loss, 'log': tensorboard_logs}\n\n    def configure_optimizers(self):\n        return torch.optim.Adam(self.parameters(), lr=self.params.fit.lr,\n                                betas=(self.params.fit.b1, self.params.fit.b2),\n                                weight_decay=self.params.fit.weight_decay)\n\n    def _get_dl(self, for_what):\n        files = self.train_files if for_what == 'train' else self.val_files\n        cache_file = f'{self.params.model_directory}/__cache_{str(files[0]).split(\"/\")[-3]}_{for_what}.npy'\n        ds = Task2Dataset(files,\n                          n_mels=self.params.feature.n_mels,\n                          frames=self.params.feature.frames,\n                          n_fft=self.params.feature.n_fft,\n                          hop_length=self.params.feature.hop_length,\n                          power=self.params.feature.power,\n                          transform=ToTensor1ch(device=device),\n                          normalize=self.normalize,\n                          cache_to=cache_file)\n        return torch.utils.data.DataLoader(ds, batch_size=self.params.fit.batch_size,\n                          shuffle=(self.params.fit.shuffle if for_what == 'train' else False))\n\n    def train_dataloader(self):\n        return self._get_dl('train')\n\n    def val_dataloader(self):\n        return self._get_dl('val')\nclass Task2ImageLightning(Task2CNNVAELightning):\n    \"\"\"Task2 PyTorch Lightning class, for training only.\"\"\"\n\n    def __init__(self, device, model, params, preprocessed_file, normalize=True):\n        super().__init__(device, model, params, files=None, normalize=normalize)\n        self.params = params\n        self.normalize = normalize\n        self.model = model\n        self.mseloss = torch.nn.MSELoss()\n\n        to_tensor = ToTensor1ch(device=self.device, image=True)\n        self.trn_ds = Task2ImageDataset(preprocessed_file, transform=to_tensor,\n                                      normalize=normalize)\n        self.val_ds = Task2ImageDataset(preprocessed_file, transform=to_tensor,\n                                      normalize=normalize, random=False)\n        train_index = self.trn_ds.get_index_by_pct()\n        self.trn_ds.train_split(train_index)\n        self.val_ds.val_split(train_index)\n\n    def _get_dl(self, for_what):\n        ds = self.trn_ds if for_what == 'train' else self.val_ds\n        return torch.utils.data.DataLoader(ds, batch_size=self.params.fit.batch_size,\n                          shuffle=(self.params.fit.shuffle if for_what == 'train' else False))","metadata":{"execution":{"iopub.status.busy":"2022-06-12T19:41:05.631675Z","iopub.execute_input":"2022-06-12T19:41:05.632086Z","iopub.status.idle":"2022-06-12T19:41:05.751021Z","shell.execute_reply.started":"2022-06-12T19:41:05.632043Z","shell.execute_reply":"2022-06-12T19:41:05.749998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_cnnae():\n        for target in types:\n\n            print(f'==== Start training [{target}] with {torch.cuda.device_count()} GPU(s). ====')\n            model = CNNAE().to(device)\n            task2 = Task2ImageLightning(device, model, params_cnnae, preprocessed_file=data[target][1])\n            #summary(device, model)\n            trainer = pl.Trainer(max_epochs=params_cnnae.fit.epochs,\n                                 gpus=torch.cuda.device_count())\n            trainer.fit(task2)\n\n            model_file = f'{params_cnnae.model_directory}/model_{target}.pth'\n            torch.save(task2.model.state_dict(), model_file)\n            print(f'saved {model_file}.\\n')\ntrain_cnnae()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T19:41:28.295699Z","iopub.execute_input":"2022-06-12T19:41:28.296109Z","iopub.status.idle":"2022-06-12T20:07:20.213294Z","shell.execute_reply.started":"2022-06-12T19:41:28.296072Z","shell.execute_reply":"2022-06-12T20:07:20.212068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Task2ImageDatasetForTest(Task2ImageDataset):\n    def get_test_batch_x(self, file_index, n_mels=params_cnnae.feature.n_mels, steps=20):\n        log_mel_spectrogram = self.X[file_index]\n        vector_array_size = (log_mel_spectrogram.shape[1] - n_mels + 1) // steps\n        vector_array = np.zeros((vector_array_size, n_mels, n_mels))\n        for t in range(vector_array_size):\n            vector_array[t] = log_mel_spectrogram[:, t*steps:t*steps+n_mels]\n        return vector_array","metadata":{"execution":{"iopub.status.busy":"2022-06-12T20:07:30.528499Z","iopub.execute_input":"2022-06-12T20:07:30.529155Z","iopub.status.idle":"2022-06-12T20:07:30.606230Z","shell.execute_reply.started":"2022-06-12T20:07:30.529116Z","shell.execute_reply":"2022-06-12T20:07:30.604999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def file_to_vector_array_2d(file_name,\n                         n_mels=128,\n                         steps=20,\n                         n_fft=1024,\n                         hop_length=512,\n                         power=2.0):\n    \"\"\"\n    convert file_name to a 2d vector array.\n    file_name : str\n        target .wav file\n    return : np.array( np.array( float ) )\n        vector array\n        * dataset.shape = (dataset_size, feature_vector_length)\n    \"\"\"\n    # 02 generate melspectrogram using librosa\n    y, sr = com.file_load(file_name)\n    mel_spectrogram = librosa.feature.melspectrogram(y=y,\n                                                     sr=sr,\n                                                     n_fft=n_fft,\n                                                     hop_length=hop_length,\n                                                     n_mels=n_mels,\n                                                     power=power)\n\n    # 03 convert melspectrogram to log mel energy\n    log_mel_spectrogram = 20.0 / power * np.log10(mel_spectrogram + sys.float_info.epsilon)\n\n    # 04 calculate total vector size\n    vector_array_size = (log_mel_spectrogram.shape[1] - n_mels + 1) // steps\n\n    # 06 generate feature vectors by concatenating multiframes\n    vector_array = np.zeros((vector_array_size, n_mels, n_mels))\n    for t in range(vector_array_size):\n        vector_array[t] = log_mel_spectrogram[:, t*steps:t*steps+n_mels]\n\n    return vector_array","metadata":{"execution":{"iopub.status.busy":"2022-06-12T20:07:34.856184Z","iopub.execute_input":"2022-06-12T20:07:34.857072Z","iopub.status.idle":"2022-06-12T20:07:34.933479Z","shell.execute_reply.started":"2022-06-12T20:07:34.857030Z","shell.execute_reply":"2022-06-12T20:07:34.932527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_cnnae():\n    # make output result directory\n    os.makedirs(params_cnnae.result_directory, exist_ok=True)\n\n    # load base directory\n    dirs = com.select_dirs(param=params, mode=params_cnnae.mode)\n\n    # initialize lines in csv for AUC and pAUC\n    csv_lines = []\n\n    # PyTorch version specific...\n    to_tensor = pytorch_common.ToTensor1ch(image=True)\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n    # loop of the base directory\n    for idx, target_dir in enumerate(dirs):\n        print(\"\\n===========================\")\n        print(\"[{idx}/{total}] {dirname}\".format(dirname=target_dir, idx=idx+1, total=len(dirs)))\n        machine_type = os.path.split(target_dir)[1]\n\n        print(\"============== MODEL LOAD ==============\")\n        # set model path\n        model_file = \"{model}/model_{machine_type}.pth\".format(model=params_cnnae.model_directory,\n                                                               machine_type=machine_type)\n\n        # load model file\n        if not os.path.exists(model_file):\n            com.logger.error(\"{} model not found \".format(machine_type))\n            sys.exit(-1)\n        com.logger.info(\"loading model: {}\".format(model_file))\n        model = CNNAE().to(device)\n        pytorch_common.load_weights(model, model_file)\n        model.eval()\n\n        if params_cnnae.mode:\n            # results by type\n            csv_lines.append([machine_type])\n            csv_lines.append([\"id\", \"AUC\", \"pAUC\"])\n            performance = []\n\n        machine_id_list = get_machine_id_list_for_test(target_dir)\n\n        for id_str in machine_id_list:\n            # load test file\n            test_files, y_true = test_file_list_generator(target_dir, id_str, mode=params_cnnae.mode)\n            print(machine_type)\n            print(f'test preprocesssed file: {data[machine_type][0]}')\n            ds = Task2ImageDatasetForTest(data[machine_type][0], random=False, normalize=True)\n\n            # setup anomaly score file path\n            anomaly_score_csv = \"{result}/anomaly_score_{machine_type}_{id_str}.csv\".format(\n                                                                                     result=params_cnnae.result_directory,\n                                                                                     machine_type=machine_type,\n                                                                                     id_str=id_str)\n            anomaly_score_list = []\n\n            print(\"\\n============== BEGIN TEST FOR A MACHINE ID ==============\")\n            y_pred = [0. for k in test_files]\n            for file_idx, file_path in tqdm(enumerate(test_files), total=len(test_files)):\n                try:\n                    data_ = file_to_vector_array_2d(file_path,\n                                                    n_mels=params_cnnae.feature.n_mels,\n                                                    n_fft=params_cnnae.feature.n_fft,\n                                                    hop_length=params_cnnae.feature.hop_length,\n                                                    power=params_cnnae.feature.power)\n                    data_ = pytorch_common.normalize_0to1(data_)\n                    with torch.no_grad():\n                        yhat = model(to_tensor(data_)).cpu().detach().numpy().reshape(data_.shape)\n                        errors = np.mean(np.square(data_ - yhat), axis=1)\n                    y_pred[file_idx] = np.mean(errors)\n                    anomaly_score_list.append([os.path.basename(file_path), y_pred[file_idx]])\n                except:\n                    com.logger.error(\"file broken!!: {}\".format(file_path))\n                    sys.exit(-1)\n\n            # save anomaly score\n            save_csv(save_file_path=anomaly_score_csv, save_data=anomaly_score_list)\n            com.logger.info(\"anomaly score result ->  {}\".format(anomaly_score_csv))\n\n            if params_cnnae.mode:\n                # append AUC and pAUC to lists\n                auc = metrics.roc_auc_score(y_true, y_pred)\n                p_auc = metrics.roc_auc_score(y_true, y_pred, max_fpr=params_cnnae.max_fpr)\n                csv_lines.append([id_str.split(\"_\", 1)[1], auc, p_auc])\n                performance.append([auc, p_auc])\n                com.logger.info(\"AUC : {}\".format(auc))\n                com.logger.info(\"pAUC : {}\".format(p_auc))\n\n            print(\"\\n============ END OF TEST FOR A MACHINE ID ============\")\n\n        if params_cnnae.mode:\n            # calculate averages for AUCs and pAUCs\n            averaged_performance = np.mean(np.array(performance, dtype=float), axis=0)\n            csv_lines.append([\"Average\"] + list(averaged_performance))\n            csv_lines.append([])\n\n    if params_cnnae.mode:\n        # output results\n        result_path = \"{result}/{file_name}\".format(result=params_cnnae.result_directory, file_name=params_cnnae.result_file)\n        com.logger.info(\"AUC and pAUC results -> {}\".format(result_path))\n        save_csv(save_file_path=result_path, save_data=csv_lines)\n\ntest_cnnae()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T20:16:55.199647Z","iopub.execute_input":"2022-06-12T20:16:55.200068Z","iopub.status.idle":"2022-06-12T20:17:34.745257Z","shell.execute_reply.started":"2022-06-12T20:16:55.200026Z","shell.execute_reply":"2022-06-12T20:17:34.744320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def upto_6digits(cell):\n    if not cell[0].isdigit(): return cell\n    return f'{float(cell):.6f}'\n\nif params_cnnae.mode:\n    with open('./result/result.csv') as f:\n        for l in f.readlines():\n            l = l.strip() #replace('\\n', '')\n            if ',' not in l:\n                print(l)\n                continue\n            ls = l.split(',')\n            print(f'{ls[0]}\\t\\t{upto_6digits(ls[1])}\\t\\t{upto_6digits(ls[2])}')","metadata":{"execution":{"iopub.status.busy":"2022-06-12T20:18:19.166701Z","iopub.execute_input":"2022-06-12T20:18:19.167087Z","iopub.status.idle":"2022-06-12T20:18:19.243675Z","shell.execute_reply.started":"2022-06-12T20:18:19.167054Z","shell.execute_reply":"2022-06-12T20:18:19.242747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"types = ['slider']\n\ndata_files = sorted(Path(params_cnnae.eval_directory).glob('eval-dc2020t2l1*.npy'))\ndata = {t:[f for f in data_files if t in str(f)] for t in types}","metadata":{"execution":{"iopub.status.busy":"2022-06-12T20:18:26.678414Z","iopub.execute_input":"2022-06-12T20:18:26.679306Z","iopub.status.idle":"2022-06-12T20:18:26.753720Z","shell.execute_reply.started":"2022-06-12T20:18:26.679266Z","shell.execute_reply":"2022-06-12T20:18:26.752741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params_cnnae.mode = False\ntrain_cnnae()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T20:18:34.318503Z","iopub.execute_input":"2022-06-12T20:18:34.318917Z","iopub.status.idle":"2022-06-12T20:44:14.394802Z","shell.execute_reply.started":"2022-06-12T20:18:34.318883Z","shell.execute_reply":"2022-06-12T20:44:14.393918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params_cnnae.mode = False\ntest_cnnae()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T20:44:27.028719Z","iopub.execute_input":"2022-06-12T20:44:27.029117Z","iopub.status.idle":"2022-06-12T20:44:58.410635Z","shell.execute_reply.started":"2022-06-12T20:44:27.029082Z","shell.execute_reply":"2022-06-12T20:44:58.409767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = pd.read_csv('./result/anomaly_score_slider_id_01.csv', delimiter=',', header=None)\ndf3 = pd.read_csv('./result/anomaly_score_slider_id_03.csv', delimiter=',', header=None)\ndf5 = pd.read_csv('./result/anomaly_score_slider_id_05.csv', delimiter=',', header=None)\n#df1 = pd.read_csv('drive/MyDrive/AML/result/anomaly_score_slider_id_01.csv', delimiter=',', header = None)\n#df3 = pd.read_csv('drive/MyDrive/AML/result/anomaly_score_slider_id_03.csv', delimiter=',', header = None)\n#df5 = pd.read_csv('drive/MyDrive/AML/result/anomaly_score_slider_id_05.csv', delimiter=',', header = None)\n\ndfs = [df1, df3, df5]\neval_df = pd.concat(dfs)\neval_df = eval_df.rename(columns={0: \"file_name\", 1: \"anomaly_score\"})\n\neval_df.to_csv('/kaggle/working/submissionCNNAE.csv',index = False)\n#eval_df.to_csv('drive/MyDrive/AML/result/CNNAE.csv',index = False)","metadata":{"execution":{"iopub.status.busy":"2022-06-12T20:45:08.187785Z","iopub.execute_input":"2022-06-12T20:45:08.188178Z","iopub.status.idle":"2022-06-12T20:45:08.280890Z","shell.execute_reply.started":"2022-06-12T20:45:08.188141Z","shell.execute_reply":"2022-06-12T20:45:08.280003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-12T20:45:12.795803Z","iopub.execute_input":"2022-06-12T20:45:12.796208Z","iopub.status.idle":"2022-06-12T20:45:12.874373Z","shell.execute_reply.started":"2022-06-12T20:45:12.796172Z","shell.execute_reply":"2022-06-12T20:45:12.873526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}