{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Table of content:**\n\n1. Introduction\n\n2. Importing libraries and Reading the Dataset\n\n3. Data Analysis & Preprocessing\n    - 3.1 Sound waves\n    - 3.2 Short Time Fourier Tranform\n    - 3.3 Signal distribution\n    - 3.3 Preprocessing\n\n4. Model:\n    - 4.1 VAE\n    - 4.2 Dense Auto Encoder\n\n\n5. Testing and submission","metadata":{}},{"cell_type":"markdown","source":"**1. Introduction**","metadata":{}},{"cell_type":"markdown","source":"**2. Importing libraries and Reading the Dataset**","metadata":{}},{"cell_type":"code","source":"!pip install torchsummary\n!pip install dl-cliche","metadata":{"execution":{"iopub.status.busy":"2022-05-24T12:04:44.280135Z","iopub.execute_input":"2022-05-24T12:04:44.280493Z","iopub.status.idle":"2022-05-24T12:05:07.867532Z","shell.execute_reply.started":"2022-05-24T12:04:44.280406Z","shell.execute_reply":"2022-05-24T12:05:07.866442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from shutil import copyfile\n\n# copy our file into the working directory (make sure it has .py suffix)\ncopyfile(src = \"../input/dcase2020-task2-baseline/pytorch_common.py\", dst = \"../working/pytorch_common.py\")\ncopyfile(src = \"../input/dcase2020-task2-baseline/common.py\", dst = \"../working/common.py\")\n\n# import all our functions\nimport common as com\nimport pytorch_common\nfrom pytorch_common import *\n\n# public modules\nimport torch\nimport torch.nn.functional as F\nimport torchsummary\nimport pytorch_lightning as pl\nimport random\nimport librosa\nimport librosa.display\nimport os\nimport glob\nimport csv\nimport re\nimport itertools\nimport sys\nimport path\nimport numpy\nimport seaborn as sns\nfrom torch import nn\nfrom tqdm import tqdm\nfrom sklearn import metrics\nfrom dlcliche.utils import *\nfrom dlcliche.notebook import *\nfrom dlcliche.utils import (\n    sys, random, Path, np, plt, EasyDict,\n    ensure_folder, deterministic_everything,\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T12:05:07.870860Z","iopub.execute_input":"2022-05-24T12:05:07.871234Z","iopub.status.idle":"2022-05-24T12:05:13.300075Z","shell.execute_reply.started":"2022-05-24T12:05:07.871191Z","shell.execute_reply":"2022-05-24T12:05:13.299065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# development dataset\nDATA_ROOT=Path(\"../input/eurecom-aml-2022-challenge-2/dev_data/dev_data\")\ntypes = [t.name for t in sorted(DATA_ROOT.glob('*')) if t.is_dir()]\nprint('Machine types:', types)\n\ndf = pd.DataFrame()\ndf['file'] = sorted(DATA_ROOT.glob('*/*/*.wav'))\ndf['type'] = df.file.map(lambda f: f.parent.parent.name)\ndf['split'] = df.file.map(lambda f: f.parent.name)\ndf['id'] = df.file.map(lambda s: str(s).split('/')[-1].split('_')[-2])","metadata":{"execution":{"iopub.status.busy":"2022-05-24T12:05:13.301728Z","iopub.execute_input":"2022-05-24T12:05:13.302025Z","iopub.status.idle":"2022-05-24T12:05:13.977647Z","shell.execute_reply.started":"2022-05-24T12:05:13.301993Z","shell.execute_reply":"2022-05-24T12:05:13.976861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-24T12:05:13.980168Z","iopub.execute_input":"2022-05-24T12:05:13.980506Z","iopub.status.idle":"2022-05-24T12:05:14.052370Z","shell.execute_reply.started":"2022-05-24T12:05:13.980469Z","shell.execute_reply":"2022-05-24T12:05:14.051501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**3. Data Analysis**","metadata":{}},{"cell_type":"code","source":"agg = df.groupby(['id', 'split']).agg('count')\nfig = plt.figure(figsize=(12.0, 6.0))\ng = sns.barplot(x=\"id\", y=\"file\", hue=\"split\", data=agg.reset_index())\nplt.xlabel(\"machine id\")\nplt.ylabel(\"number of files\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-24T12:05:14.054006Z","iopub.execute_input":"2022-05-24T12:05:14.054563Z","iopub.status.idle":"2022-05-24T12:05:14.381164Z","shell.execute_reply.started":"2022-05-24T12:05:14.054521Z","shell.execute_reply":"2022-05-24T12:05:14.380332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"First, we can see that number of training data are imbalanced between machines. Now, we are going to compare the characteristics of one normal signal and one anormal signal.","metadata":{}},{"cell_type":"code","source":"print(df['file'][1000], df['file'][0])","metadata":{"execution":{"iopub.status.busy":"2022-05-24T12:05:14.382316Z","iopub.execute_input":"2022-05-24T12:05:14.382561Z","iopub.status.idle":"2022-05-24T12:05:14.440682Z","shell.execute_reply.started":"2022-05-24T12:05:14.382534Z","shell.execute_reply":"2022-05-24T12:05:14.439755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get sound signal\nnormal_signal_file = df['file'][1000]\nabnormal_signal_file =df['file'][0]\nnormal_signal, _ = librosa.load(normal_signal_file)\nabnormal_signal, sr = librosa.load(abnormal_signal_file)","metadata":{"execution":{"iopub.status.busy":"2022-05-24T12:05:14.442239Z","iopub.execute_input":"2022-05-24T12:05:14.442733Z","iopub.status.idle":"2022-05-24T12:05:15.831127Z","shell.execute_reply.started":"2022-05-24T12:05:14.442690Z","shell.execute_reply":"2022-05-24T12:05:15.830395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prop_cycle = plt.rcParams[\"axes.prop_cycle\"]\ncolors = prop_cycle.by_key()[\"color\"]","metadata":{"execution":{"iopub.status.busy":"2022-05-24T12:05:15.834324Z","iopub.execute_input":"2022-05-24T12:05:15.834605Z","iopub.status.idle":"2022-05-24T12:05:15.891896Z","shell.execute_reply.started":"2022-05-24T12:05:15.834574Z","shell.execute_reply":"2022-05-24T12:05:15.890960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**3.1 Sound waves**","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(24, 6))\n\nplt.subplot(1, 3, 1)\nlibrosa.display.waveshow(normal_signal,sr=sr,alpha=0.5,color=colors[1],linewidth=0.5,label=\"Machine #id_00 - Normal Signal\",\n)\nplt.title(\"Normal Signal\")\n\nplt.subplot(1, 3, 2)\nlibrosa.display.waveshow(abnormal_signal,sr=sr,alpha=0.6,color=colors[5],linewidth=0.5,label=\"Machine #id_00 - Abnormal Signal\",\n)\nplt.title(\"Abnormal Signal\")\n\nplt.subplot(1, 3, 3)\nlibrosa.display.waveshow(abnormal_signal,sr=sr,alpha=0.6,color=colors[5],linewidth=0.5,label=\"Abnormal Signal\",\n)\nlibrosa.display.waveshow(normal_signal,sr=sr,alpha=0.5,color=colors[1],linewidth=0.5,label=\"Normal Signal\",\n)\nplt.title(\"Both Signals\")\n\nfig.suptitle(\"Machine #id_00 - 2D Representation of the Wave Forms\", fontsize=16)\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-24T12:05:15.893678Z","iopub.execute_input":"2022-05-24T12:05:15.894681Z","iopub.status.idle":"2022-05-24T12:05:18.713830Z","shell.execute_reply.started":"2022-05-24T12:05:15.894639Z","shell.execute_reply":"2022-05-24T12:05:18.713080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In this case, we can see that the abnormal signal has a bigger amplitude and is less regular the the normal one.","metadata":{}},{"cell_type":"markdown","source":"**3.2 Short Time Fourier Tranform**","metadata":{}},{"cell_type":"code","source":"n_fft = 2048\nnormal_stft = np.abs(\n    librosa.stft(normal_signal[:n_fft], n_fft=n_fft, hop_length=n_fft + 1)\n)\nabnormal_stft = np.abs(\n    librosa.stft(abnormal_signal[:n_fft], n_fft=n_fft, hop_length=n_fft + 1)\n)\n\nfig = plt.figure(figsize=(12, 6))\nplt.plot(\n    normal_stft, color=colors[1], alpha=0.6, label=\"Machine #id_00 - Normal Signal\"\n)\nplt.plot(\n    abnormal_stft, color=colors[5], alpha=0.6, label=\"Machine #id_00 - Abnormal Signal\"\n)\nplt.title(\n    f\"Short Time Fourier Transform for the first {n_fft / sr * 1000:.0f}ms Window\"\n)\nplt.xlabel(\"Frequency (Hz)\")\nplt.ylabel(\"Amplitude\")\nplt.legend()\nplt.xlim(0, 200)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-24T12:05:18.716445Z","iopub.execute_input":"2022-05-24T12:05:18.716689Z","iopub.status.idle":"2022-05-24T12:05:19.033989Z","shell.execute_reply.started":"2022-05-24T12:05:18.716662Z","shell.execute_reply":"2022-05-24T12:05:19.033335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see a very hight amplitude of the abnormal signal that goes above 2.5 in the lowest frequencies, when the normal signal never goes above1.4.","metadata":{}},{"cell_type":"markdown","source":"**3.3 Signal distribution**","metadata":{}},{"cell_type":"code","source":"normal = 0\nanormal = 0\nfor i in range (df.shape[0]):\n    n = df['file'][i].stem.find('normal')\n    a = df['file'][i].stem.find('anomaly')\n    if n > -1:\n        normal += 1\n    if a > -1:\n        anormal += 1\nprint(normal, anormal, normal+anormal, df.shape[0] )\nprint('The purcentages of anormal signal over normal signal on the entire dataset', anormal*100/normal, '%')","metadata":{"execution":{"iopub.status.busy":"2022-05-24T12:05:19.035030Z","iopub.execute_input":"2022-05-24T12:05:19.035690Z","iopub.status.idle":"2022-05-24T12:05:19.163470Z","shell.execute_reply.started":"2022-05-24T12:05:19.035656Z","shell.execute_reply":"2022-05-24T12:05:19.162689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# development dataset\nDATA_ROOT=Path(\"../input/eurecom-aml-2022-challenge-2/dev_data/dev_data\")\ntypes = [t.name for t in sorted(DATA_ROOT.glob('*')) if t.is_dir()]\nprint('Machine types:', types)\n\ndf = pd.DataFrame()\ndf['file'] = sorted(DATA_ROOT.glob('*/*/*.wav'))\ndf['type'] = df.file.map(lambda f: f.parent.parent.name)\ndf['split'] = df.file.map(lambda f: f.parent.name)\ndf['id'] = df.file.map(lambda s: str(s).split('/')[-1].split('_')[-2])\n\nagg = df.groupby(['id', 'split']).agg('count')\nfig = plt.figure(figsize=(12.0, 6.0))\ng = sns.barplot(x=\"id\", y=\"file\", hue=\"split\", data=agg.reset_index())\nplt.xlabel(\"machine id\")\nplt.ylabel(\"number of files\")\nplt.show()\nagg.transpose()","metadata":{"execution":{"iopub.status.busy":"2022-05-24T12:05:19.164739Z","iopub.execute_input":"2022-05-24T12:05:19.164971Z","iopub.status.idle":"2022-05-24T12:05:19.552160Z","shell.execute_reply.started":"2022-05-24T12:05:19.164934Z","shell.execute_reply":"2022-05-24T12:05:19.551267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(' There is ', 968+968+434, ' training data', ' so because we know that the train dataset is only composed of normal signal, the proportion of anormal signal in the test set is ', anormal*100/(normal-968+968+434))","metadata":{"execution":{"iopub.status.busy":"2022-05-24T12:05:19.553470Z","iopub.execute_input":"2022-05-24T12:05:19.554069Z","iopub.status.idle":"2022-05-24T12:05:19.612993Z","shell.execute_reply.started":"2022-05-24T12:05:19.554028Z","shell.execute_reply":"2022-05-24T12:05:19.612097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Because there is 75% of normal data in our set, a dummy model that only predicted 1, saying that signals are normals, would produce 75% of accuracy. Thus, our objective is to do better than 75%.","metadata":{}},{"cell_type":"markdown","source":"**3.4 Preprocessing**","metadata":{}},{"cell_type":"code","source":"# development dataset\nDATA_ROOT=Path(\"../input/eurecom-aml-2022-challenge-2/dev_data/dev_data\")\ntypes = [t.name for t in sorted(DATA_ROOT.glob('*')) if t.is_dir()]\nprint('Machine types:', types)\n\ndf = pd.DataFrame()\ndf['file'] = sorted(DATA_ROOT.glob('*/*/*.wav'))\ndf['type'] = df.file.map(lambda f: f.parent.parent.name)\ndf['split'] = df.file.map(lambda f: f.parent.name)\ndf['id'] = df.file.map(lambda s: str(s).split('/')[-1].split('_')[-2])\n\nagg = df.groupby(['id', 'split']).agg('count')\nfig = plt.figure(figsize=(12.0, 6.0))\ng = sns.barplot(x=\"id\", y=\"file\", hue=\"split\", data=agg.reset_index())\nplt.xlabel(\"machine id\")\nplt.ylabel(\"number of files\")\nplt.show()\nagg.transpose()","metadata":{"execution":{"iopub.status.busy":"2022-05-24T12:05:19.614464Z","iopub.execute_input":"2022-05-24T12:05:19.614969Z","iopub.status.idle":"2022-05-24T12:05:20.000235Z","shell.execute_reply.started":"2022-05-24T12:05:19.614912Z","shell.execute_reply":"2022-05-24T12:05:19.999327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We first tried to do some undersampling in oder to have a number of input more balanced between machines. Unfortunately, results were worse so we used another technique to deal with the problem.","metadata":{}},{"cell_type":"code","source":"nmin = df['id'].value_counts().min()\ndf_ = df.groupby('id').apply(lambda x: x.sample(nmin)).reset_index(drop=True)\ndf_ = df_[df_['split'] == 'train']\n\nagg = df_.groupby(['id']).agg('count')\nfig = plt.figure(figsize=(12.0, 6.0))\ng = sns.barplot(x=\"id\", y=\"file\", data=agg.reset_index())\nplt.xlabel(\"machine id\")\nplt.ylabel(\"number of files\")\nplt.show()\nagg.transpose()\n\ndf_ = df_['file'].tolist()","metadata":{"execution":{"iopub.status.busy":"2022-05-24T12:05:20.002088Z","iopub.execute_input":"2022-05-24T12:05:20.002420Z","iopub.status.idle":"2022-05-24T12:05:20.258698Z","shell.execute_reply.started":"2022-05-24T12:05:20.002382Z","shell.execute_reply":"2022-05-24T12:05:20.258119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**4. Model**","metadata":{}},{"cell_type":"markdown","source":"**4.1 VAE**","metadata":{}},{"cell_type":"markdown","source":"We used the Variational auto-encoder solution. We ran the code and the first thing that we observed was that the result for the last machine, the one with the less training data, wasn't as good as the result for other machines.","metadata":{}},{"cell_type":"code","source":"class VAE(nn.Module):\n    \"\"\"Mostly borrowed from PyTorch example.\n    Thanks to https://github.com/pytorch/examples/blob/master/vae/main.py\n    \"\"\"\n\n    def __init__(self, device, x_dim=640, h_dim=400, z_dim=20):\n        super().__init__()\n        self.x_dim = x_dim\n\n        self.fc11 = nn.Linear(x_dim, h_dim)\n        self.fc12 = nn.Linear(h_dim, h_dim)\n        self.fc21 = nn.Linear(h_dim, z_dim)\n        self.fc22 = nn.Linear(h_dim, z_dim)\n        self.fc31 = nn.Linear(z_dim, h_dim)\n        self.fc32 = nn.Linear(h_dim, h_dim)\n        self.fc4 = nn.Linear(h_dim, x_dim)\n\n    def encode(self, x):\n        x = F.relu(self.fc11(x))\n        x = F.relu(self.fc12(x))\n        return self.fc21(x), self.fc22(x)\n\n    def reparameterize(self, mu, logvar):\n        std = torch.exp(0.5*logvar)\n        eps = torch.randn_like(std)\n        return mu + eps*std\n\n    def decode(self, z):\n        h = F.relu(self.fc31(z))\n        h = F.relu(self.fc32(h))\n        return torch.sigmoid(self.fc4(h))\n\n    def forward_all(self, x):\n        mu, logvar = self.encode(x.view(-1, self.x_dim))\n        z = self.reparameterize(mu, logvar)\n        return self.decode(z), z, mu, logvar\n\n    def forward(self, x):\n        yhat, _, _, _ = self.forward_all(x)\n        return yhat\n\n\ndef VAE_loss_function(recon_x, x, mu, logvar, reconst_loss='mse', a_RECONST=1., a_KLD=1., x_dim=640):\n    \"\"\"Loss function for VAE which consists of reconstruction and KL divergence losses.\n    Thanks to https://github.com/pytorch/examples/blob/master/vae/main.py\n\n    You can also balance weights for each loss, just to see what if KLD loss is stronger, etc.\n\n    Args:\n        reconst_loss: Reconstruction loss calculation: 'mse' or 'bce'\n        a_RECONST: Weight for reconstruction loss.\n        a_KLD: Weight for KLD loss.\n    \"\"\"\n\n    func = (F.mse_loss if reconst_loss == 'mse'\n            else F.binary_cross_entropy if reconst_loss == 'bce'\n            else 'Unknown reconst_loss')\n    RECONST = func(recon_x, x.view(-1, x_dim), reduction='sum')\n\n    # see Appendix B from VAE paper:\n    # Kingma and Welling. Auto-Encoding Variational Bayes. ICLR, 2014\n    # https://arxiv.org/abs/1312.6114\n    # 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)\n    KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())\n\n    return RECONST*a_RECONST + KLD*a_KLD\n","metadata":{"execution":{"iopub.status.busy":"2022-05-24T09:34:40.059146Z","iopub.execute_input":"2022-05-24T09:34:40.059433Z","iopub.status.idle":"2022-05-24T09:34:40.131006Z","shell.execute_reply.started":"2022-05-24T09:34:40.059406Z","shell.execute_reply":"2022-05-24T09:34:40.129897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {\n# inout directory\n\"dev_directory\" : \"../input/eurecom-aml-2022-challenge-2/dev_data/dev_data\",\n\"eval_directory\" : \"../input/eurecom-aml-2022-challenge-2/eval_data/eval_data\",\n\"model_directory\": \"./model\",\n\"result_directory\": \"./result\",\n\"result_file\": \"result.csv\",\n\"max_fpr\" : 0.1,\n\"mode\" : True, # mode=True for development dataset, mode=False for evaluation dataset\n\n# preprocessing for mel-spectrogram\n\"feature\": {\n\"n_mels\": 128,\n\"frames\" : 5,\n\"n_fft\": 1024,\n\"hop_length\": 512,\n\"power\": 2.0},\n\n# training\n\"fit\":{\n\"lr\": 0.0005,\n\"b1\": 0.9,\n\"b2\": 0.999,\n\"weight_decay\": 0.0,\n\"epochs\" : 100\n\"batch_size\" : 1500,\n\"shuffle\" : True,\n\"validation_split\" : 0.1,\n\"verbose\" : 1},\n    \n# model architecture\n\"VAE\":{\n\"x_dim\": 640,\n\"h_dim\": 400,\n\"z_dim\": 20}\n    \n}\nparams = EasyDict(params)\n\n# create working directory\nensure_folder(params.model_directory)\n\n# test directories\ndirs = com.select_dirs(param=params, mode=params.mode)\n\n# fix random seeds\ndeterministic_everything(2020, pytorch=True)\n\n# PyTorch device\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2022-05-24T09:45:37.448639Z","iopub.execute_input":"2022-05-24T09:45:37.448939Z","iopub.status.idle":"2022-05-24T09:45:37.545281Z","shell.execute_reply.started":"2022-05-24T09:45:37.448901Z","shell.execute_reply":"2022-05-24T09:45:37.544421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Task2Dataset(torch.utils.data.Dataset):\n    \"\"\"PyTorch dataset class for task2. Caching to a file supported.\n\n    Args:\n        n_mels, frames, n_fft, hop_length, power, transform: Audio conversion settings.\n        normalize: Normalize data value range from [-90, 24] to [0, 1] for VAE, False by default.\n        cache_to: Cache filename or None by default, use this for your iterative development.\n    \"\"\"\n\n    def __init__(self, files, n_mels, frames, n_fft, hop_length, power, transform,\n                 normalize=False, cache_to=None):\n        self.transform = transform\n        self.files = files\n        self.n_mels, self.frames, self.n_fft = n_mels, frames, n_fft\n        self.hop_length, self.power = hop_length, power\n        # load cache or convert all the data for the first time\n        if cache_to is not None and Path(cache_to).exists():\n            com.logger.info(f'Loading cached {Path(cache_to).name}')\n            self.X = np.load(cache_to)\n        else:\n            self.X = com.list_to_vector_array(self.files,\n                             n_mels=self.n_mels,\n                             frames=self.frames,\n                             n_fft=self.n_fft,\n                             hop_length=self.hop_length,\n                             power=self.power)\n            if cache_to is not None:\n                np.save(cache_to, self.X)\n\n        if normalize:\n            self.X = normalize_0to1(self.X)\n\n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, index):\n        x = self.X[index]\n        x = self.transform(x)\n        return x, x\n\n\n# VAE Training class\nclass Task2VAELightning(pl.LightningModule):\n    \"\"\"Task2 PyTorch Lightning class, for training only.\"\"\"\n\n    def __init__(self, device, model, params, files, normalize=False):\n        super().__init__()\n        self.params = params\n        self.normalize = normalize\n        self.model = model\n        self.mseloss = torch.nn.MSELoss()\n        # split data files\n        if files is not None:\n            n_val = int(params.fit.validation_split * len(files))\n            self.val_files = random.sample(files, n_val)\n            self.train_files = [f for f in files if f not in self.val_files]\n\n    def forward(self, x):\n        return self.model(x)\n    \n    def training_step(self, batch, batch_nb):\n        x, y = batch\n        y_hat, z, mu, logvar = self.model.forward_all(x)\n        loss = VAE_loss_function(recon_x=y_hat, x=x, mu=mu, logvar=logvar,\n                                 reconst_loss='mse',\n                                 a_RECONST=1.,\n                                 a_KLD=.01)\n        tensorboard_logs = {'train_loss': loss}\n        return {'loss': loss, 'log': tensorboard_logs}\n\n    def validation_step(self, batch, batch_nb):\n        x, y = batch\n        y_hat = self.forward(x)\n        return {'val_loss': self.mseloss(y_hat, y)}\n\n    def validation_epoch_end(self, outputs):\n        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n        tensorboard_logs = {'val_loss': avg_loss}\n        return {'avg_val_loss': avg_loss, 'log': tensorboard_logs}\n\n    def configure_optimizers(self):\n        return torch.optim.Adam(self.parameters(), lr=self.params.fit.lr,\n                                betas=(self.params.fit.b1, self.params.fit.b2),\n                                weight_decay=self.params.fit.weight_decay)\n\n    def _get_dl(self, for_what):\n        files = self.train_files if for_what == 'train' else self.val_files\n        cache_file = f'{self.params.model_directory}/__cache_{str(files[0]).split(\"/\")[-3]}_{for_what}.npy'\n        ds = Task2Dataset(files,\n                          n_mels=self.params.feature.n_mels,\n                          frames=self.params.feature.frames,\n                          n_fft=self.params.feature.n_fft,\n                          hop_length=self.params.feature.hop_length,\n                          power=self.params.feature.power,\n                          transform=ToTensor1ch(device=device),\n                          normalize=self.normalize,\n                          cache_to=cache_file)\n        return torch.utils.data.DataLoader(ds, batch_size=self.params.fit.batch_size,\n                          shuffle=(self.params.fit.shuffle if for_what == 'train' else False))\n\n    def train_dataloader(self):\n        return self._get_dl('train')\n\n    def val_dataloader(self):\n        return self._get_dl('val')","metadata":{"execution":{"iopub.status.busy":"2022-05-24T09:34:52.871586Z","iopub.execute_input":"2022-05-24T09:34:52.871907Z","iopub.status.idle":"2022-05-24T09:34:52.949495Z","shell.execute_reply.started":"2022-05-24T09:34:52.871873Z","shell.execute_reply":"2022-05-24T09:34:52.948657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train models\n\ndef train():\n    for target_dir in dirs:\n        target = str(target_dir).split('/')[-1]\n        print(f'==== Start training [{target}] with {torch.cuda.device_count()} GPU(s). ====')\n\n        files = com.file_list_generator(target_dir)\n\n        model = VAE(device, x_dim=params.VAE.x_dim, h_dim=params.VAE.h_dim, z_dim=params.VAE.z_dim).to(device)\n        summary(device, model)\n        task2 = Task2VAELightning(device, model, params, files, normalize=True)\n        trainer = pl.Trainer(max_epochs=params.fit.epochs,\n                             gpus=torch.cuda.device_count())\n        trainer.fit(task2)\n\n        model_file = f'{params.model_directory}/model_{target}.pth'\n        torch.save(task2.model.state_dict(), model_file)\n        print(f'saved {model_file}.\\n')\n        \ntrain()","metadata":{"execution":{"iopub.status.busy":"2022-05-24T09:45:42.657311Z","iopub.execute_input":"2022-05-24T09:45:42.65844Z","iopub.status.idle":"2022-05-24T09:47:45.114237Z","shell.execute_reply.started":"2022-05-24T09:45:42.658403Z","shell.execute_reply":"2022-05-24T09:47:45.113626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We observed that the loss didn't change much after 80 epochs so we reduced this number to gain time. We then tried to tune parameters, but we still not get satisfactory results on the third machine so we decided to train and tune the model on each machine separately. Since one of the biggest difficulty is that the model runs for hours, we only compared the best results for 10 epochs thinking that the parameters that had the best results at the beggining would be likely to get best results at the end. For instance, for the third machine we used different combinaison of parameters:\n\n\"lr\": 0.001,0.0005,0.0001,0.0003\n\n\"b1\": 0.9,0.2,0.5,0.1\n\n\"b2\": 0.999,0.8\n\n\"weight_decay\": 0.0,0.1,0.5\n\n\"batch_size\" : 1000,1500\n\n\"validation_split\" : 0.1,0.5\n\nand got the best result with this combination:\n\n\"lr\": 0.0003\n\n\"b1\": 0.1\n\n\"b2\": 0.8\n\n\"weight_decay\": 0.0\n\n\"batch_size\" : 1500\n\n\"validation_split\" : 0.1\n\nWe then ran separately on each model, however the model crashes (our notebook tried to allocate more memory than is available and was restarted) so we weren't able to test it on a sufficient number of epochs.","metadata":{}},{"cell_type":"code","source":"def test(machineID):\n    # make output result directory\n    os.makedirs(params.result_directory, exist_ok=True)\n\n    # load base directory\n    dirs = com.select_dirs(param=params, mode=params.mode)\n\n    # initialize lines in csv for AUC and pAUC\n    csv_lines = []\n\n    # PyTorch version specific...\n    to_tensor = pytorch_common.ToTensor1ch()\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n    # loop of the base directory\n    for idx, target_dir in enumerate(dirs):\n        print(\"\\n===========================\")\n        print(\"[{idx}/{total}] {dirname}\".format(dirname=target_dir, idx=idx+1, total=len(dirs)))\n        machine_type = os.path.split(target_dir)[1]\n\n        print(\"============== MODEL LOAD ==============\")\n        # set model path\n        model_file = \"{model}/model_{machine_type}.pth\".format(model=params.model_directory,\n                                                               machine_type=machine_type)\n\n        # load model file\n        if not os.path.exists(model_file):\n            com.logger.error(\"{} model not found \".format(machine_type))\n            sys.exit(-1)\n        com.logger.info(\"loading model: {}\".format(model_file))\n        model = VAE(device, x_dim=params.VAE.x_dim, h_dim=params.VAE.h_dim, z_dim=params.VAE.z_dim).to(device)\n        pytorch_common.load_weights(model, model_file)\n        pytorch_common.summary(device, model)\n        model.eval()\n\n        if params.mode:\n            # results by type\n            csv_lines.append([machine_type])\n            csv_lines.append([\"id\", \"AUC\", \"pAUC\"])\n            performance = []\n\n        machine_id_list = get_machine_id_list_for_test(target_dir)\n\n        for id_str in machine_id_list:\n            if (id_str == machineID):\n                # load test file\n                test_files, y_true = test_file_list_generator(target_dir, id_str, mode=params.mode)\n\n                # setup anomaly score file path\n                anomaly_score_csv = \"{result}/anomaly_score_{machine_type}_{id_str}.csv\".format(\n                                                                                     result=params.result_directory,\n                                                                                     machine_type=machine_type,\n                                                                                     id_str=id_str)\n                anomaly_score_list = []\n\n                print(\"\\n============== BEGIN TEST FOR A MACHINE ID ==============\")\n                y_pred = [0. for k in test_files]\n                for file_idx, file_path in tqdm(enumerate(test_files), total=len(test_files)):\n                    try:\n                        data = com.file_to_vector_array(file_path,\n                                                    n_mels=params.feature.n_mels,\n                                                    frames=params.feature.frames,\n                                                    n_fft=params.feature.n_fft,\n                                                    hop_length=params.feature.hop_length,\n                                                    power=params.feature.power)\n                        data = pytorch_common.normalize_0to1(data)\n                        with torch.no_grad():\n                            yhat = model(to_tensor(data)).cpu().detach().numpy().reshape(data.shape)\n                            errors = numpy.mean(numpy.square(data - yhat), axis=1)\n                        y_pred[file_idx] = numpy.mean(errors)\n                        anomaly_score_list.append([os.path.basename(file_path), y_pred[file_idx]])\n                    except:\n                        com.logger.error(\"file broken!!: {}\".format(file_path))\n                        sys.exit(-1)\n\n                # save anomaly score\n                save_csv(save_file_path=anomaly_score_csv, save_data=anomaly_score_list)\n                com.logger.info(\"anomaly score result ->  {}\".format(anomaly_score_csv))\n\n                if params.mode:\n                    # append AUC and pAUC to lists\n                    auc = metrics.roc_auc_score(y_true, y_pred)\n                    p_auc = metrics.roc_auc_score(y_true, y_pred, max_fpr=params.max_fpr)\n                    csv_lines.append([id_str.split(\"_\", 1)[1], auc, p_auc])\n                    performance.append([auc, p_auc])\n                    com.logger.info(\"AUC : {}\".format(auc))\n                    com.logger.info(\"pAUC : {}\".format(p_auc))\n\n                print(\"\\n============ END OF TEST FOR A MACHINE ID ============\")\n\n        if params.mode:\n            # calculate averages for AUCs and pAUCs\n            averaged_performance = numpy.mean(numpy.array(performance, dtype=float), axis=0)\n            csv_lines.append([\"Average\"] + list(averaged_performance))\n            csv_lines.append([])\n\n    if params.mode:\n        # output results\n        result_path = \"{result}/{file_name}\".format(result=params.result_directory, file_name=params.result_file)\n        com.logger.info(\"AUC and pAUC results -> {}\".format(result_path))\n        save_csv(save_file_path=result_path, save_data=csv_lines)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Machine 00**","metadata":{}},{"cell_type":"code","source":"params = {\n# inout directory\n\"dev_directory\" : \"../input/eurecom-aml-2022-challenge-2/dev_data/dev_data\",\n\"eval_directory\" : \"../input/eurecom-aml-2022-challenge-2/eval_data/eval_data\",\n\"model_directory\": \"./model\",\n\"result_directory\": \"./result\",\n\"result_file\": \"result.csv\",\n\"max_fpr\" : 0.1,\n\"mode\" : True, # mode=True for development dataset, mode=False for evaluation dataset\n\n# preprocessing for mel-spectrogram\n\"feature\": {\n\"n_mels\": 128,\n\"frames\" : 5,\n\"n_fft\": 1024,\n\"hop_length\": 512,\n\"power\": 2.0},\n\n# training\n\"fit\":{\n\"lr\": 0.0005,\n\"b1\": 0.9,\n\"b2\": 0.999,\n\"weight_decay\": 0.0,\n\"epochs\" : 80,\n\"batch_size\" : 1000,\n\"shuffle\" : True,\n\"validation_split\" : 0.1,\n\"verbose\" : 1},\n    \n# model architecture\n\"VAE\":{\n\"x_dim\": 640,\n\"h_dim\": 400,\n\"z_dim\": 20}\n    \n}\nparams = EasyDict(params)\n\n# create working directory\nensure_folder(params.model_directory)\n\n# test directories\ndirs = com.select_dirs(param=params, mode=params.mode)\n\n# fix random seeds\ndeterministic_everything(2020, pytorch=True)\n\n# PyTorch device\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Train only on first machine\ndf_ = df['file'][df['id'] == '00'].tolist()\n\"\"\"\ntrain()\ntest(\"id_00\")\nparams.mode = False\ntest(\"id_01\")\nparams.mode = True\n\"\"\"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Machine 02**","metadata":{}},{"cell_type":"code","source":"params = {\n# inout directory\n\"dev_directory\" : \"../input/eurecom-aml-2022-challenge-2/dev_data/dev_data\",\n\"eval_directory\" : \"../input/eurecom-aml-2022-challenge-2/eval_data/eval_data\",\n\"model_directory\": \"./model\",\n\"result_directory\": \"./result\",\n\"result_file\": \"result.csv\",\n\"max_fpr\" : 0.1,\n\"mode\" : True, # mode=True for development dataset, mode=False for evaluation dataset\n\n# preprocessing for mel-spectrogram\n\"feature\": {\n\"n_mels\": 128,\n\"frames\" : 5,\n\"n_fft\": 1024,\n\"hop_length\": 512,\n\"power\": 2.0},\n\n# training\n\"fit\":{\n\"lr\": 0.001,\n\"b1\": 0.9,\n\"b2\": 0.8,\n\"weight_decay\": 0.0,\n\"epochs\" : 80,\n\"batch_size\" : 1000,\n\"shuffle\" : True,\n\"validation_split\" : 0.1,\n\"verbose\" : 1},\n    \n# model architecture\n\"VAE\":{\n\"x_dim\": 640,\n\"h_dim\": 400,\n\"z_dim\": 20}\n    \n}\nparams = EasyDict(params)\n\n# create working directory\nensure_folder(params.model_directory)\n\n# test directories\ndirs = com.select_dirs(param=params, mode=params.mode)\n\n# fix random seeds\ndeterministic_everything(2020, pytorch=True)\n\n# PyTorch device\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Train only on machine 02\ndf_ = df['file'][df['id'] == '02'].tolist()\n\"\"\"\ntrain()\ntest(\"id_02\")\nparams.mode = False\ntest(\"id_03\")\nparams.mode = True\n\"\"\"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Machine 04**","metadata":{}},{"cell_type":"code","source":"params = {\n# inout directory\n\"dev_directory\" : \"../input/eurecom-aml-2022-challenge-2/dev_data/dev_data\",\n\"eval_directory\" : \"../input/eurecom-aml-2022-challenge-2/eval_data/eval_data\",\n\"model_directory\": \"./model\",\n\"result_directory\": \"./result\",\n\"result_file\": \"result.csv\",\n\"max_fpr\" : 0.1,\n\"mode\" : True, # mode=True for development dataset, mode=False for evaluation dataset\n\n# preprocessing for mel-spectrogram\n\"feature\": {\n\"n_mels\": 128,\n\"frames\" : 5,\n\"n_fft\": 1024,\n\"hop_length\": 512,\n\"power\": 2.0},\n\n# training\n\"fit\":{\n\"lr\": 0.0003,\n\"b1\": 0.1,\n\"b2\": 0.8,\n\"weight_decay\": 0.0,\n\"epochs\" : 80, #100\n\"batch_size\" : 1500,\n\"shuffle\" : True,\n\"validation_split\" : 0.1,\n\"verbose\" : 1},\n    \n# model architecture\n\"VAE\":{\n\"x_dim\": 640,\n\"h_dim\": 400,\n\"z_dim\": 20}\n    \n}\nparams = EasyDict(params)\n\n# create working directory\nensure_folder(params.model_directory)\n\n# test directories\ndirs = com.select_dirs(param=params, mode=params.mode)\n\n# fix random seeds\ndeterministic_everything(2020, pytorch=True)\n\n# PyTorch device\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Train only on machine 04\ndf_ = df['file'][df['id'] == '04'].tolist()\n\"\"\"\ntrain()\ntest(\"id_04\")\nparams.mode = False\ntest(\"id_05\")\nparams.mode = True\n\"\"\"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**4.2 Dense Auto Encoder**","metadata":{}},{"cell_type":"markdown","source":"We tried to used a dense auto encoder. The main difficulty with this model is that it easily crashed when we run it, and as the VAE, it takes a long time to train.","metadata":{}},{"cell_type":"code","source":"\"\"\"\n########################################################################\n# import python-library\n########################################################################\n# default\npip install pqdm\nimport csv\nimport glob\nimport multiprocessing\nimport os\n\nimport sys\nfrom functools import partial\nfrom venv import logger\nimport re\nimport itertools\n\nimport librosa\nimport librosa.core\nimport librosa.feature\n\n# additional\nimport numpy\nimport pandas as pd\nfrom sklearn.metrics import roc_auc_score\nimport keras.models\nfrom keras.layers import Input, Dense, BatchNormalization\nfrom keras.models import Model\nfrom tqdm import tqdm\nfrom pqdm.processes import pqdm\nimport tensorflow as tf\n\ntf.random.set_seed(1234)\n\n\ndef file_load(wav_name, mono=False):\n    \"\"\"\"\"\"\n    load .wav file.\n    wav_name : str\n        target .wav file\n    sampling_rate : int\n        audio file sampling_rate\n    mono : boolean\n        When load a multi channels file and this param True, the returned data will be merged for mono data\n    return : numpy.array( float )\n    \"\"\"\"\"\"\n    try:\n        return librosa.load(wav_name, sr=None, mono=mono)\n    except:\n        logger.error(\"file_broken or not exists!! : {}\".format(wav_name))\n\n\n########################################################################\n\ndef file_load_stream(wav_name, mono=False):\n    try:\n        sr = librosa.get_samplerate(wav_name)\n        frameSize = sr\n        hoplength = frameSize // 2\n        stream = librosa.stream(wav_name,\n                                block_length=1,\n                                frame_length=frameSize,\n                                hop_length=hoplength,\n                                mono=mono)\n    except:\n        logger.error(\"file_broken or not exists!! : {}\".format(wav_name))\n    return stream\n\n\n########################################################################\n# feature extractor\n########################################################################\ntamanhoVec = []\n\n\ndef file_to_vector_array(file_name,\n                         n_mels=128,\n                         frames=5,\n                         n_fft=1024,\n                         hop_length=512,\n                         power=2.0):\n    \"\"\"\"\"\"\n    convert file_name to a vector array.\n    file_name : str\n        target .wav file\n    return : numpy.array( numpy.array( float ) )\n        vector array\n        * dataset.shape = (dataset_size, feature_vector_length)\n    \"\"\"\"\"\"\n    # 01 calculate the number of dimensions\n    dims = n_mels * frames\n    global tamanhoVec\n    # 02 generate melspectrogram using librosa\n    y, sr = file_load(file_name)\n    mel_spectrogram = librosa.feature.melspectrogram(y=y,\n                                                     sr=sr,\n                                                     n_fft=n_fft,\n                                                     hop_length=hop_length,\n                                                     n_mels=n_mels,\n                                                     power=power)\n\n    # 03 convert melspectrogram to log mel energy\n    log_mel_spectrogram = 20.0 / power * numpy.log10(mel_spectrogram + sys.float_info.epsilon)\n\n    # 04 calculate total vector size\n    vector_array_size = len(log_mel_spectrogram[0, :]) - frames + 1\n    tamanhoVec.append(vector_array_size)\n    # 05 skip too short clips\n    if vector_array_size < 1:\n        return numpy.empty((0, dims))\n\n    # 06 generate feature vectors by concatenating multiframes\n    vector_array = numpy.zeros((vector_array_size, dims))\n    for t in range(frames):\n        vector_array[:, n_mels * t: n_mels * (t + 1)] = log_mel_spectrogram[:, t: t + vector_array_size].T\n\n    return vector_array\n\n\ncontador = 0\n\n\ndef file_to_vector_array_stream(file_name,\n                                n_mels=128,\n                                frames=5,\n                                n_fft=1024,\n                                hop_length=512,\n                                power=2):\n    \"\"\"\"\"\"\n    convert file_name to a vector array.\n    file_name : str\n        target .wav file\n    return : numpy.array( numpy.array( float ) )\n        vector array\n        * dataset.shape = (dataset_size, feature_vector_length)\n    \"\"\"\"\"\"\n    # 01 calculate the number of dimensions\n\n    dims = n_mels * frames\n    global tamanhoVec\n    global contador\n    # 02 generate melspectrogram using librosa\n    stream = file_load_stream(file_name)\n    sr = librosa.get_samplerate(file_name)\n    lista = []\n    for n, y in enumerate(stream):\n        mel_spectrogram = librosa.feature.melspectrogram(y=y,\n                                                         sr=sr,\n                                                         n_fft=n_fft,\n                                                         hop_length=hop_length,\n                                                         n_mels=n_mels,\n                                                         power=power)\n\n        audio = librosa.effects.pitch_shift(y, sr, 3)\n\n        mel_spectogram_audioPitched = librosa.feature.melspectrogram(y=audio,\n                                                                     sr=sr,\n                                                                     n_fft=n_fft,\n                                                                     hop_length=hop_length,\n                                                                     n_mels=n_mels,\n                                                                     power=power)\n        # 03 convert melspectrogram\n\n        log_mel_spectrogram = 20.0 / power * numpy.log10(mel_spectrogram + sys.float_info.epsilon)\n        log_mel_spectrogramPitched = 20.0 / power * numpy.log10(mel_spectogram_audioPitched + sys.float_info.epsilon)\n        # 04 calculate total vector size\n        vector_array_size = len(log_mel_spectrogram[0, :]) - frames + 1  # aqui será len(\n        # log_mel_spectrogram[0, :])\n        vector_array_size2 = len(log_mel_spectrogramPitched[0, :]) - frames + 1\n        # vezes 2 visto que teremos a versão \"normal\" e warped!\n\n        tamanhoVec.append(vector_array_size)\n        # 05 skip too short clips\n        if vector_array_size < 1:\n            return numpy.empty((0, dims))\n        if vector_array_size2 < 1:\n            return numpy.empty((0, dims))\n\n        # 06 generate feature vectors by concatenating multiframes\n        vector_array = numpy.zeros((vector_array_size, dims))\n        vector_array2 = numpy.zeros((vector_array_size2, dims))\n        for t in range(frames):\n            vector_array[:, n_mels * t: n_mels * (t + 1)] = log_mel_spectrogram[:, t: t + vector_array_size].T\n\n        for t in range(frames):\n            vector_array2[:, n_mels * t: n_mels * (t + 1)] = mel_spectogram_audioPitched[:, t: t + vector_array_size2].T\n\n        lista.append(vector_array)\n        lista.append(vector_array2)\n        contador += 1\n\n    lista2 = numpy.asarray(lista)\n    lista2 = lista2.reshape(lista2.shape[0] * lista2.shape[1], lista2.shape[2])\n    return lista2\n\n\ndef file_to_vector_array_stream_test_data(file_name,\n                                          n_mels=128,\n                                          frames=5,\n                                          n_fft=1024,\n                                          hop_length=512,\n                                          power=1):\n    \"\"\"\"\"\"\n    convert file_name to a vector array.\n    file_name : str\n        target .wav file\n    return : numpy.array( numpy.array( float ) )\n        vector array\n        * dataset.shape = (dataset_size, feature_vector_length)\n    \"\"\"\"\"\"\n    # 01 calculate the number of dimensions\n\n    dims = n_mels * frames\n    global tamanhoVec\n    global contador\n    # 02 generate melspectrogram using librosa\n    stream = file_load_stream(file_name)\n    sr = librosa.get_samplerate(file_name)\n    lista = []\n    for n, y in enumerate(stream):\n        mel_spectrogram = librosa.feature.melspectrogram(y=y,\n                                                         sr=sr,\n                                                         n_fft=n_fft,\n                                                         hop_length=hop_length,\n                                                         n_mels=n_mels,\n                                                         power=power)\n\n        # 03 convert melspectrogram\n\n        log_mel_spectrogram = 20.0 / power * numpy.log10(mel_spectrogram + sys.float_info.epsilon)\n\n        # 04 calculate total vector size\n        vector_array_size = len(log_mel_spectrogram[0, :]) - frames + 1  # aqui será len(\n        # log_mel_spectrogram[0, :])\n\n        # vezes 2 visto que teremos a versão \"normal\" e warped!\n\n        tamanhoVec.append(vector_array_size)\n        # 05 skip too short clips\n        if vector_array_size < 1:\n            return numpy.empty((0, dims))\n\n        # 06 generate feature vectors by concatenating multiframes\n        vector_array = numpy.zeros((vector_array_size, dims))\n\n        for t in range(frames):\n            vector_array[:, n_mels * t: n_mels * (t + 1)] = log_mel_spectrogram[:, t: t + vector_array_size].T\n\n        lista.append(vector_array)\n\n        contador += 1\n\n    lista2 = numpy.asarray(lista)\n    lista2 = lista2.reshape(lista2.shape[0] * lista2.shape[1], lista2.shape[2])\n    return lista2\n\n\ndef numpy_dataset(train_files):\n    from tqdm import tqdm\n    par_file = partial(file_to_vector_array)\n    tq_files = tqdm(train_files)\n    np_data = numpy.concatenate(list(map(par_file, tq_files)))\n    return np_data\n\n\ndef numpy_dataset_stream(train_files):\n    par_file = partial(file_to_vector_array_stream)\n    # tq_files = tqdm(train_files)\n    from pqdm.processes import pqdm\n    import multiprocessing\n    d = pqdm(train_files, par_file, n_jobs=multiprocessing.cpu_count())\n    list_audio = []\n    for i in d:\n        list_audio.append(i)\n    newlist = numpy.asarray(list_audio)\n    newlist = newlist.reshape(newlist.shape[0] * newlist.shape[1], newlist.shape[2])\n    return newlist\n\n\ndef numpy_dataset_stream_noaugment(train_files):\n    par_file = partial(file_to_vector_array_stream_test_data)\n\n    from pqdm.processes import pqdm\n    import multiprocessing\n    d = pqdm(train_files, par_file, n_jobs=multiprocessing.cpu_count())\n    list_audio = []\n    for i in d:\n        list_audio.append(i)\n    newlist = numpy.asarray(list_audio)\n    newlist = newlist.reshape(newlist.shape[0] * newlist.shape[1], newlist.shape[2])\n    return newlist\n\n\ndef get_model(inputDim, name_for_model=\"\"):\n    \"\"\"\"\"\"\n    define the keras model\n    the model based on the simple dense auto encoder\n    (128*128*128*128*8*128*128*128*128)\n    \"\"\"\"\"\"\n    # x (100*100*100*100*64*100*100*100*100) activation elu, optimizer adam base x spectogram = 2' --> best para o\n    # slider!\n    # ver o caso para o toycar! com elu e o que acontecerá?\n    inputLayer = Input(shape=(inputDim,))\n\n    h = Dense(512)(inputLayer)  # estava 128\n    h = BatchNormalization()(h)\n    h = keras.layers.ReLU()(h)\n\n    h = Dense(512)(h)\n    h = BatchNormalization()(h)\n    h = keras.layers.ReLU()(h)\n\n    h = Dense(512)(h)\n    h = BatchNormalization()(h)\n    h = keras.layers.ReLU()(h)\n\n    h = Dense(512)(h)\n    h = BatchNormalization()(h)\n    h = keras.layers.ReLU()(h)\n\n    h = Dense(8)(h)\n    h = BatchNormalization()(h)\n    h = keras.layers.ReLU()(h)\n\n    h = Dense(512)(h)\n    h = BatchNormalization()(h)\n    h = keras.layers.ReLU()(h)\n\n    h = Dense(512)(h)\n    h = BatchNormalization()(h)\n    h = keras.layers.ReLU()(h)\n\n    h = Dense(512)(h)\n    h = BatchNormalization()(h)\n    h = keras.layers.ReLU()(h)\n\n    h = Dense(512)(h)\n    h = BatchNormalization()(h)\n    h = keras.layers.ReLU()(h)\n\n    h = Dense(inputDim)(h)\n\n    vae2 = Model(inputLayer, h, name='AE' + name_for_model)\n\n    vae2.compile(optimizer='adam', loss=\"mse\")\n    return vae2\n\n\ndef get_machine_id_list_for_test(target_dir,\n                                 dir_name=\"test\",\n                                 ext=\"wav\"):\n    \"\"\"\"\"\"\n    target_dir : str\n        base directory path of \"dev_data\" or \"eval_data\"\n    test_dir_name : str (default=\"test\")\n        directory containing test data\n    ext : str (default=\"wav)\n        file extension of audio files\n    return :\n        machine_id_list : list [ str ]\n            list of machine IDs extracted from the names of test files\n    \"\"\"\"\"\"\n    # create test files\n    dir_path = os.path.abspath(\"{dir}/{dir_name}/*.{ext}\".format(dir=target_dir, dir_name=dir_name, ext=ext))\n    file_paths = sorted(glob.glob(dir_path))\n    # extract id\n    machine_id_list = sorted(list(set(itertools.chain.from_iterable(\n        [re.findall('id_[0-9][0-9]', ext_id) for ext_id in file_paths]))))\n    return machine_id_list\n\n\ndef eval_model(target_dir, model, machine_type):\n    ### load parameters\n    # initialize lines in csv for AUC and pAUC\n    csv_lines = [[machine_type], [\"id\", \"AUC\", \"pAUC\"]]\n    # results by type\n    performance = []\n\n    machine_id_list = get_machine_id_list_for_test(target_dir)\n\n    for id_str in machine_id_list:\n        # load test file\n        test_files, y_true = test_file_list_generator(target_dir, id_str)\n\n        # setup anomaly score file path\n        anomaly_score_csv = (\"{result}/anomaly_score_{machine_type}_{id_str}.csv\"\n                             .format(result=\"models/results/\",\n                                     machine_type=machine_type,\n                                     id_str=id_str))\n\n        anomaly_score_list = []\n        print(\"\\n============== BEGIN TEST FOR A MACHINE ID ==============\")\n        y_pred = [0. for k in test_files]\n        for file_idx, file_path in tqdm(enumerate(test_files),\n                                        total=len(test_files)):\n            try:\n                data = file_to_vector_array(file_path)\n                data = data.reshape(data.shape[0], data.shape[1], 1)\n                from sklearn.metrics.pairwise import cosine_similarity\n                errors = numpy.mean(numpy.square(data - model.predict(data)),\n                                    axis=1)\n                y_pred[file_idx] = numpy.mean(errors)\n                anomaly_score_list.append([os.path.basename(file_path),\n                                           y_pred[file_idx]])\n            except:\n                print(\"file broken!!: {}\".format(file_path))\n\n        # save anomaly score\n        save_csv(save_file_path=anomaly_score_csv, save_data=anomaly_score_list)\n        print(\"anomaly score result ->  {}\".format(anomaly_score_csv))\n\n        # append AUC and pAUC to lists\n        auc = roc_auc_score(y_true, y_pred)\n        p_auc = roc_auc_score(y_true, y_pred, max_fpr=0.1)\n        csv_lines.append([id_str.split(\"_\", 1)[1], auc, p_auc])\n        performance.append([auc, p_auc])\n        print(\"AUC : {}\".format(auc))\n        print(\"pAUC : {}\".format(p_auc))\n\n        print(\"\\n============ END OF TEST FOR A MACHINE ID ============\")\n\n    return csv_lines\n\n\ndef eval_model_stream(target_dir, model, machine_type):\n    ### load parameters\n    # initialize lines in csv for AUC and pAUC\n    csv_lines = [[machine_type], [\"id\", \"AUC\", \"pAUC\"]]\n    # results by type\n    performance = []\n\n    machine_id_list = get_machine_id_list_for_test(target_dir)\n\n    for id_str in machine_id_list:\n        # load test file\n        test_files, y_true = test_file_list_generator(target_dir, id_str)\n\n        # setup anomaly score file path\n        anomaly_score_csv = (\"{result}/anomaly_score_{machine_type}_{id_str}.csv\"\n                             .format(result=\"models/results/\",\n                                     machine_type=machine_type,\n                                     id_str=id_str))\n\n        anomaly_score_list = []\n        print(\"\\n============== BEGIN TEST FOR A MACHINE ID ==============\")\n        y_pred = [0. for k in test_files]\n        for file_idx, file_path in tqdm(enumerate(test_files),\n                                        total=len(test_files)):\n            data = file_to_vector_array_stream_test_data(file_path)\n            errors = numpy.mean(numpy.square(data - model.predict(data)),\n                                axis=1)\n            y_pred[file_idx] = numpy.mean(errors)\n            anomaly_score_list.append([os.path.basename(file_path),\n                                       y_pred[file_idx]])\n\n        # save anomaly score\n        save_csv(save_file_path=anomaly_score_csv, save_data=anomaly_score_list)\n        print(\"anomaly score result ->  {}\".format(anomaly_score_csv))\n\n        # append AUC and pAUC to lists\n        auc = roc_auc_score(y_true, y_pred)\n        p_auc = roc_auc_score(y_true, y_pred, max_fpr=0.1)\n        csv_lines.append([id_str.split(\"_\", 1)[1], auc, p_auc])\n        performance.append([auc, p_auc])\n        print(\"AUC : {}\".format(auc))\n        print(\"pAUC : {}\".format(p_auc))\n\n        print(\"\\n============ END OF TEST FOR A MACHINE ID ============\")\n\n    return csv_lines\n\n\ndef eval_model_stream_evaluation(target_dir, model, machine_type):\n    ### load parameters\n    # initialize lines in csv for AUC and pAUC\n    csv_lines = [[machine_type], [\"id\", \"AUC\", \"pAUC\"]]\n    # results by type\n    performance = []\n\n    machine_id_list = get_machine_id_list_for_test(target_dir)\n\n    for id_str in machine_id_list:\n        # load test file\n        test_files, y_true = test_file_list_generator(target_dir, id_str)\n\n        # setup anomaly score file path\n        anomaly_score_csv = (\"{result}/anomaly_score_{machine_type}_{id_str}.csv\"\n                             .format(result=\"models/results/\",\n                                     machine_type=machine_type,\n                                     id_str=id_str))\n\n        anomaly_score_list = []\n        print(\"\\n============== BEGIN TEST FOR A MACHINE ID ==============\")\n        y_pred = [0. for k in test_files]\n        for file_idx, file_path in tqdm(enumerate(test_files),\n                                        total=len(test_files)):\n            data = file_to_vector_array_stream_test_data(file_path)\n            errors = numpy.mean(numpy.square(data - model.predict(data)),\n                                axis=1)\n            y_pred[file_idx] = numpy.mean(errors)\n            anomaly_score_list.append([os.path.basename(file_path),\n                                       y_pred[file_idx]])\n\n        # save anomaly score\n        save_csv(save_file_path=anomaly_score_csv, save_data=anomaly_score_list)\n        print(\"anomaly score result ->  {}\".format(anomaly_score_csv))\n\n        print(\"\\n============ END OF TEST FOR A MACHINE ID ============\")\n\n    return csv_lines\n\n\ndef test_file_list_generator(target_dir, id_name, dir_name=\"test\",\n                             prefix_normal=\"normal\", prefix_anomaly=\"anomaly\",\n                             ext=\"wav\"):\n    normal_files = sorted(\n        glob.glob(\"{dir}/{dir_name}/{prefix_normal}_{id_name}*.{ext}\"\n                  .format(dir=target_dir, dir_name=dir_name,\n                          prefix_normal=prefix_normal,\n                          id_name=id_name, ext=ext)))\n\n    normal_labels = numpy.zeros(len(normal_files))\n    anomaly_files = sorted(\n        glob.glob(\"{dir}/{dir_name}/{prefix_anomaly}_{id_name}*.{ext}\"\n                  .format(dir=target_dir, dir_name=dir_name,\n                          prefix_anomaly=prefix_anomaly,\n                          id_name=id_name, ext=ext)))\n\n    anomaly_labels = numpy.ones(len(anomaly_files))\n    files = numpy.concatenate((normal_files, anomaly_files), axis=0)\n    labels = numpy.concatenate((normal_labels, anomaly_labels), axis=0)\n\n    if len(files) == 0:\n        print(\"no_wav_file!!\")\n\n    return files, labels\n\n\ndef save_csv(save_file_path,\n             save_data):\n    with open(save_file_path, \"w\", newline=\"\") as f:\n        writer = csv.writer(f, lineterminator='\\n', delimiter=';')\n        writer.writerows(save_data)\n\n\nif __name__ == '__main__':\n    multiprocessing.freeze_support()\n    path = os.getcwd()\n    print(path, \"path\")\n    folders = []\n    foldersForChallenge = [\"dev_data/dev_data/slider\"]  # machines\n    \n\n    for f in foldersForChallenge:\n        callbacks_listv1 = [\n            keras.callbacks.EarlyStopping(\n                monitor='val_loss',\n                mode='min',\n                patience=10,\n                min_delta=0.01,\n                restore_best_weights=True,\n\n            ), keras.callbacks.ModelCheckpoint(\n                monitor='val_loss',\n                mode='min',\n                filepath=path + \"/modelBest\" + \"_\" + f + \".H5\",\n                save_best_only=True,\n                verbose=1\n            )]\n        files_Read = []\n        print(path)\n        os.chdir(\"../input/eurecom-aml-2022-challenge-2/\" + f + \"/train/\")\n        for file in glob.glob(\"*.wav\"):\n            # print(file)\n            files_Read.append(file)\n        #print(files_Read)\n        files_Read0 = files_Read[:134]\n        files_Read2 = files_Read[967:967+134]\n        files_Read4 = files_Read[:-134]\n        files_Read = files_Read0 + files_Read2 + files_Read4\n\n        b = numpy_dataset_stream_noaugment(files_Read)  # data read numpy_dataset_stream_noaugment --> no augmentation\n        AE = get_model(inputDim=b.shape[1], name_for_model=f)\n        AE.fit(b, b, batch_size=512, epochs=2, callbacks=callbacks_listv1, validation_split=0.3)\n        os.chdir(path)\n        AEBest = keras.models.load_model(\"modelBest\" + \"_\" + f + \".H5\")\n        dataAE = eval_model_stream(path + \"/\" + f, AEBest, f)\n        data_csv = pd.DataFrame(dataAE, columns=dataAE[1])[2:]\n        modelEvals = os.getcwd() + \"/models/results\"\n        # mudar o path para o eval para onde querem guardar!\n        data_csv.to_csv(modelEvals + \"/\" + f + \".csv\")\n        print(\"AUC Average\", numpy.mean(data_csv[\"AUC\"]))\n        print(\"pAUC Average\", numpy.mean(data_csv[\"pAUC\"]))\n\"\"\"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**5. Testing and submission**","metadata":{}},{"cell_type":"code","source":"def test():\n    # make output result directory\n    os.makedirs(params.result_directory, exist_ok=True)\n\n    # load base directory\n    dirs = com.select_dirs(param=params, mode=params.mode)\n\n    # initialize lines in csv for AUC and pAUC\n    csv_lines = []\n\n    # PyTorch version specific...\n    to_tensor = pytorch_common.ToTensor1ch()\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n    # loop of the base directory\n    for idx, target_dir in enumerate(dirs):\n        print(\"\\n===========================\")\n        print(\"[{idx}/{total}] {dirname}\".format(dirname=target_dir, idx=idx+1, total=len(dirs)))\n        machine_type = os.path.split(target_dir)[1]\n\n        print(\"============== MODEL LOAD ==============\")\n        # set model path\n        model_file = \"{model}/model_{machine_type}.pth\".format(model=params.model_directory,\n                                                               machine_type=machine_type)\n\n        # load model file\n        if not os.path.exists(model_file):\n            com.logger.error(\"{} model not found \".format(machine_type))\n            sys.exit(-1)\n        com.logger.info(\"loading model: {}\".format(model_file))\n        model = VAE(device, x_dim=params.VAE.x_dim, h_dim=params.VAE.h_dim, z_dim=params.VAE.z_dim).to(device)\n        pytorch_common.load_weights(model, model_file)\n        pytorch_common.summary(device, model)\n        model.eval()\n\n        if params.mode:\n            # results by type\n            csv_lines.append([machine_type])\n            csv_lines.append([\"id\", \"AUC\", \"pAUC\"])\n            performance = []\n\n        machine_id_list = get_machine_id_list_for_test(target_dir)\n\n        for id_str in machine_id_list:\n            # load test file\n            test_files, y_true = test_file_list_generator(target_dir, id_str, mode=params.mode)\n\n            # setup anomaly score file path\n            anomaly_score_csv = \"{result}/anomaly_score_{machine_type}_{id_str}.csv\".format(\n                                                                                     result=params.result_directory,\n                                                                                     machine_type=machine_type,\n                                                                                     id_str=id_str)\n            anomaly_score_list = []\n\n            print(\"\\n============== BEGIN TEST FOR A MACHINE ID ==============\")\n            y_pred = [0. for k in test_files]\n            for file_idx, file_path in tqdm(enumerate(test_files), total=len(test_files)):\n                try:\n                    data = com.file_to_vector_array(file_path,\n                                                    n_mels=params.feature.n_mels,\n                                                    frames=params.feature.frames,\n                                                    n_fft=params.feature.n_fft,\n                                                    hop_length=params.feature.hop_length,\n                                                    power=params.feature.power)\n                    data = pytorch_common.normalize_0to1(data)\n                    with torch.no_grad():\n                        yhat = model(to_tensor(data)).cpu().detach().numpy().reshape(data.shape)\n                        errors = numpy.mean(numpy.square(data - yhat), axis=1)\n                    y_pred[file_idx] = numpy.mean(errors)\n                    anomaly_score_list.append([os.path.basename(file_path), y_pred[file_idx]])\n                except:\n                    com.logger.error(\"file broken!!: {}\".format(file_path))\n                    sys.exit(-1)\n\n            # save anomaly score\n            save_csv(save_file_path=anomaly_score_csv, save_data=anomaly_score_list)\n            com.logger.info(\"anomaly score result ->  {}\".format(anomaly_score_csv))\n\n            if params.mode:\n                # append AUC and pAUC to lists\n                auc = metrics.roc_auc_score(y_true, y_pred)\n                p_auc = metrics.roc_auc_score(y_true, y_pred, max_fpr=params.max_fpr)\n                csv_lines.append([id_str.split(\"_\", 1)[1], auc, p_auc])\n                performance.append([auc, p_auc])\n                com.logger.info(\"AUC : {}\".format(auc))\n                com.logger.info(\"pAUC : {}\".format(p_auc))\n\n            print(\"\\n============ END OF TEST FOR A MACHINE ID ============\")\n\n        if params.mode:\n            # calculate averages for AUCs and pAUCs\n            averaged_performance = numpy.mean(numpy.array(performance, dtype=float), axis=0)\n            csv_lines.append([\"Average\"] + list(averaged_performance))\n            csv_lines.append([])\n\n    if params.mode:\n        # output results\n        result_path = \"{result}/{file_name}\".format(result=params.result_directory, file_name=params.result_file)\n        com.logger.info(\"AUC and pAUC results -> {}\".format(result_path))\n        save_csv(save_file_path=result_path, save_data=csv_lines)\n\ntest()","metadata":{"execution":{"iopub.status.busy":"2022-05-24T09:47:45.116548Z","iopub.execute_input":"2022-05-24T09:47:45.117155Z","iopub.status.idle":"2022-05-24T09:48:54.187836Z","shell.execute_reply.started":"2022-05-24T09:47:45.117112Z","shell.execute_reply":"2022-05-24T09:48:54.185209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def upto_6digits(cell):\n    if not cell[0].isdigit(): return cell\n    return f'{float(cell):.6f}'\n\nif params.mode:\n    with open('./result/result.csv') as f:\n        for l in f.readlines():\n            l = l.strip() #replace('\\n', '')\n            if ',' not in l:\n                print(l)\n                continue\n            ls = l.split(',')\n            print(f'{ls[0]}\\t\\t{upto_6digits(ls[1])}\\t\\t{upto_6digits(ls[2])}')","metadata":{"execution":{"iopub.status.busy":"2022-05-24T09:48:54.189274Z","iopub.execute_input":"2022-05-24T09:48:54.189609Z","iopub.status.idle":"2022-05-24T09:48:54.311863Z","shell.execute_reply.started":"2022-05-24T09:48:54.18955Z","shell.execute_reply":"2022-05-24T09:48:54.310823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# re-training on evaluation dataset\nparams.mode = False\n# train() # due to the memory restriction, restart the kernel and train on only evaluation dataset","metadata":{"execution":{"iopub.status.busy":"2022-05-24T09:48:54.314341Z","iopub.execute_input":"2022-05-24T09:48:54.314717Z","iopub.status.idle":"2022-05-24T09:48:54.397921Z","shell.execute_reply.started":"2022-05-24T09:48:54.314677Z","shell.execute_reply":"2022-05-24T09:48:54.39717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test()","metadata":{"execution":{"iopub.status.busy":"2022-05-24T09:48:54.39917Z","iopub.execute_input":"2022-05-24T09:48:54.399585Z","iopub.status.idle":"2022-05-24T09:49:50.241337Z","shell.execute_reply.started":"2022-05-24T09:48:54.399534Z","shell.execute_reply":"2022-05-24T09:49:50.240357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\ndf1 = pd.read_csv('./result/anomaly_score_slider_id_01.csv', delimiter=',', header=None)\ndf3 = pd.read_csv('./result/anomaly_score_slider_id_03.csv', delimiter=',', header=None)\ndf5 = pd.read_csv('./result/anomaly_score_slider_id_05.csv', delimiter=',', header=None)\n\ndfs = [df1, df3, df5]\neval_df = pd.concat(dfs)\neval_df = eval_df.rename(columns={0: \"file_name\", 1: \"anomaly_score\"})\n\neval_df.to_csv('./result/baseline_submission.csv',index = False)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tar -zcvf eval_df.tar.gz /kaggle/working","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import HTML\ndef create_download_link(title = \"Download CSV file\", filename = \"baseline_submission.csv\"):  \n    html = '<a href={filename}>{title}</a>'\n    html = html.format(title=title,filename=filename)\n    return HTML(html)\n\ncreate_download_link(filename='baseline_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}