{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Anomaly Sound Detection","metadata":{}},{"cell_type":"markdown","source":"We tried different models (Dense autoencoder, Variational autoencoder and Convolutional autoencoder). The last one gave us the best result, and the following notebook explains how we implemented this model. We ran the model locally, with some files that we were not able to upload on the kaggle platform. However, as we based our model architecture on the one provided by the repository https://github.com/AlexandrineRibeiro/DCASE-2020-Task-2, these files can be found there. ","metadata":{}},{"cell_type":"code","source":"import os\nimport glob\nimport sys\nimport numpy\nimport itertools\nimport re\nimport csv\nfrom tqdm import tqdm\nfrom common_cae import *\nimport common_cae as com\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nimport keras.models\nfrom keras.models import Model\nfrom keras.layers import Input, BatchNormalization, Activation, Reshape, Flatten\nfrom keras.layers import Conv2D, Cropping2D, Conv2DTranspose, Dense\nfrom keras.utils.vis_utils import plot_model\nfrom keras.backend import int_shape\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping\nfrom tensorflow import set_random_seed\nfrom sklearn.externals.joblib import load, dump\nfrom sklearn import metrics","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define model: Convolutional autoencoder","metadata":{}},{"cell_type":"code","source":"def get_data_shape(layer):\n    return tuple(int_shape(layer)[1:])\n\n\ndef get_model(inputDim, latentDim):\n    \"\"\"\n    define the keras model\n    the model based on the simple convolutional auto encoder \n    \"\"\"\n    input_img = Input(shape=(inputDim[0], inputDim[1], 1))  # adapt this if using 'channels_first' image data format\n\n    # encoder\n    x = Conv2D(32, (5, 5),strides=(1,2), padding='same')(input_img)   #32x128 -> 32x64\n    x = BatchNormalization()(x)\n    x = Activation('relu')(x)\n    x = Conv2D(64, (5, 5),strides=(1,2), padding='same')(x)           #32x32\n    x = BatchNormalization()(x)\n    x = Activation('relu')(x)\n    x = Conv2D(128, (5, 5),strides=(2,2), padding='same')(x)          #16x16\n    x = BatchNormalization()(x)\n    x = Activation('relu')(x)\n    x = Conv2D(256, (3, 3),strides=(2,2), padding='same')(x)          #8x8\n    x = BatchNormalization()(x)\n    x = Activation('relu')(x)\n    x = Conv2D(512, (3, 3),strides=(2,2), padding='same')(x)          #4x4\n    x = BatchNormalization()(x)\n    x = Activation('relu')(x)\n\n    volumeSize = int_shape(x)\n    # at this point the representation size is latentDim i.e. latentDim-dimensional\n    x = Conv2D(latentDim, (4,4), strides=(1,1), padding='valid')(x)\n    encoded = Flatten()(x)\n     \n    # decoder\n    x = Dense(volumeSize[1] * volumeSize[2] * volumeSize[3])(encoded) \n    x = Reshape((volumeSize[1], volumeSize[2], 512))(x)                #4x4\n\n    x = Conv2DTranspose(256, (3, 3),strides=(2,2), padding='same')(x)  #8x8\n    x = BatchNormalization()(x)\n    x = Activation('relu')(x)\n    x = Conv2DTranspose(128, (3, 3),strides=(2,2), padding='same')(x)  #16x16   \n    x = BatchNormalization()(x)\n    x = Activation('relu')(x)\n    x = Conv2DTranspose(64, (5, 5),strides=(2,2), padding='same')(x)   #32x32\n    x = BatchNormalization()(x)\n    x = Activation('relu')(x)\n    x = Conv2DTranspose(32, (5, 5),strides=(1,2), padding='same')(x)   #32x64\n    x = BatchNormalization()(x)\n    x = Activation('relu')(x)\n    \n    decoded = Conv2DTranspose(1, (5, 5),strides=(1,2), padding='same')(x) \n\n    return Model(inputs=input_img, outputs=decoded)\n\n\ndef load_model(file_path):\n    return keras.models.load_model(file_path)\n\n\ndef plot(model):\n    plot_model(model, to_file='model_plot.png', show_shapes=True, show_layer_names=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize loss","metadata":{}},{"cell_type":"code","source":"class visualizer(object):\n    def __init__(self):\n        import matplotlib.pyplot as plt\n        self.plt = plt\n        self.fig = self.plt.figure(figsize=(30, 10))\n        self.plt.subplots_adjust(wspace=0.3, hspace=0.3)\n\n    def loss_plot(self, loss, val_loss):\n        \"\"\"\n        Plot loss curve.\n\n        loss : list [ float ]\n            training loss time series.\n        val_loss : list [ float ]\n            validation loss time series.\n\n        return   : None\n        \"\"\"\n        ax = self.fig.add_subplot(1, 1, 1)\n        ax.cla()\n        ax.plot(loss)\n        ax.plot(val_loss)\n        ax.set_title(\"Model loss\")\n        ax.set_xlabel(\"Epoch\")\n        ax.set_ylabel(\"Loss\")\n        ax.legend([\"Train\", \"Validation\"], loc=\"upper right\")\n\n    def save_figure(self, name):\n        \"\"\"\n        Save figure.\n\n        name : str\n            save png file path.\n\n        return : None\n        \"\"\"\n        self.plt.savefig(name)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Parameters","metadata":{}},{"cell_type":"code","source":"param = com.yaml_load()\nparam","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train","metadata":{}},{"cell_type":"code","source":"class DataGenerator(keras.utils.Sequence):\n    'Generates data for Keras'\n    def __init__(self, list_IDs, batch_size=32, dim=(32,128), shuffle=True, step=8):\n        'Initialization'\n        self.dim = dim\n        self.batch_size = batch_size\n        self.list_IDs = list_IDs\n        self.shuffle = shuffle\n\n        self.data = np.load(self.list_IDs[0] , mmap_mode='r')\n        \n        self.step = step\n        self.indexes_start = np.arange(self.data.shape[1]-self.dim[0]+self.step, step=self.step)\n        self.max = len(self.indexes_start)\n        self.indexes = np.arange(self.data.shape[0])\n        \n        self.indexes = np.repeat(self.indexes, self.max )\n        self.indexes_start = np.repeat(self.indexes_start, self.data.shape[0])\n    \n        self.on_epoch_end()\n\n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        return int(np.floor(self.data.shape[0] * self.max  / self.batch_size))\n\n    def __getitem__(self, index):\n        'Generate one batch of data'\n        # Generate indexes of the batch\n\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n        indexes_start = self.indexes_start[index*self.batch_size:(index+1)*self.batch_size]\n\n        # Generate data\n        X = self.__data_generation(indexes, indexes_start).reshape((self.batch_size, *self.dim, 1))\n\n        return X, X\n\n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        \n        if self.shuffle == True:\n            np.random.shuffle(self.indexes)\n            np.random.shuffle(self.indexes_start)\n\n\n    def __data_generation(self, indexes, index_start):\n        'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)\n        # Initialization\n        X = np.empty((self.batch_size, *self.dim))\n\n        # Generate data\n        for i, (id_file, id_start) in enumerate(zip(indexes, index_start)):\n\n            x = self.data[id_file,]\n            length, mels = x.shape\n\n            start = id_start\n\n            start = min(start, length - self.dim[0])\n            \n            # crop part of sample\n            crop = x[start:start+self.dim[0], :]\n\n            X[i,] = crop\n        return X","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check mode\n# \"development\": mode == True\n# \"evaluation\": mode == False\nmode, target = True, \"slider\"\n\n# fix a random seed\nset_random_seed(88)\n\nif mode is None:\n    sys.exit(-1)\n\n# make output directory\nos.makedirs(param[\"model_directory\"], exist_ok=True)\n\n# initialize the visualizer\nvisualizer = visualizer()\n\n# load base_directory list\ndirs = com.select_dirs(param=param, mode=mode, target=target)\n\n\n# loop of the base directory (machine types)\nfor idx, target_dir in enumerate(dirs):\n    print(\"\\n===========================\")\n    print(\"[{idx}/{total}] {dirname}\".format(dirname=target_dir, idx=idx+1, total=len(dirs)))\n\n    # set path\n    machine_type = os.path.split(target_dir)[1]\n    model_file_path = \"{model}/model_{machine_type}.hdf5\".format(model=param[\"model_directory\"],\n                                                                machine_type=machine_type)\n    best_model_filepath = \"{model}/bestmodel_{machine_type}_\".format(model=param[\"model_directory\"],\n                                                                machine_type=machine_type)\n    history_img = \"{model}/history__{machine_type}.png\".format(model=param[\"model_directory\"],\n                                                                machine_type=machine_type)\n    features_file_path = \"{features}/{machine_type}\".format(features=param[\"features_directory\"],\n                                                                machine_type=machine_type)\n    features_dir_path = os.path.abspath(features_file_path)\n\n\n    if os.path.exists(model_file_path):\n        com.logger.info(\"model exists\")\n        continue\n\n\n    # get features\n    # get npy files list (features files)\n    list_files_npy_train = file_list_generator(features_dir_path, dir_name=\"train\", ext=\"npy\")\n    list_files_npy_val = file_list_generator(features_dir_path, dir_name=\"val\", ext=\"npy\")\n\n    if len(list_files_npy_train)==0 or len(list_files_npy_val)==0:\n        com.logger.exception(\"no_npy_files!!\")\n        sys.exit(-1)  \n\n\n    shape0_feat = param[\"autoencoder\"][\"shape0\"]\n    shape1_feat = param[\"feature\"][\"n_mels\"]\n\n    # load data \n    gen_train = DataGenerator(list_files_npy_train, batch_size=param[\"fit\"][\"batch_size\"], dim=(shape0_feat,shape1_feat), step=param[\"step\"])\n    gen_val = DataGenerator(list_files_npy_val,  batch_size=param[\"fit\"][\"batch_size\"], dim=(shape0_feat,shape1_feat), shuffle=False, step=param[\"step\"])\n\n\n    # train model\n    print(\"============== MODEL TRAINING ==============\")\n\n    # checkpoint\n    model_checkpoint = ModelCheckpoint(best_model_filepath+\"{epoch:02d}.hdf5\", monitor='val_loss', verbose=1, save_best_only=True)\n    early = EarlyStopping(monitor='val_loss', mode='min', patience=10, min_delta=0.0001)\n\n    # create model\n    model = get_model((shape0_feat, shape1_feat), param[\"autoencoder\"][\"latentDim\"])\n    model.summary()\n\n\n    #train model\n    model.compile(**param[\"fit\"][\"compile\"])\n    history = model.fit_generator(gen_train, \n                        validation_data=gen_val,\n                        epochs=param[\"fit\"][\"epochs\"], \n                        verbose=param[\"fit\"][\"verbose\"],\n                        callbacks=[model_checkpoint, early])\n\n    visualizer.loss_plot(history.history[\"loss\"], history.history[\"val_loss\"])\n    visualizer.save_figure(history_img)\n    model.save(model_file_path)\n    com.logger.info(\"save_model -> {}\".format(model_file_path))\n    print(\"============== END TRAINING ==============\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test","metadata":{}},{"cell_type":"code","source":"# useful functions\ndef save_csv(save_file_path,\n             save_data):\n    with open(save_file_path, \"w\", newline=\"\") as f:\n        writer = csv.writer(f, lineterminator='\\n')\n        writer.writerows(save_data)\n\n\ndef get_machine_id_list_for_test(target_dir,\n                                 dir_name=\"test\",\n                                 ext=\"wav\"):\n    # create test files\n    dir_path = os.path.abspath(\"{dir}/{dir_name}/*.{ext}\".format(dir=target_dir, dir_name=dir_name, ext=ext))\n    file_paths = sorted(glob.glob(dir_path))\n    # extract id\n    machine_id_list = sorted(list(set(itertools.chain.from_iterable(\n        [re.findall('id_[0-9][0-9]', ext_id) for ext_id in file_paths]))))\n    return machine_id_list\n\n\ndef test_file_list_generator(target_dir,\n                             id_name,\n                             dir_name=\"test\",\n                             prefix_normal=\"normal\",\n                             prefix_anomaly=\"anomaly\",\n                             ext=\"wav\"):\n    com.logger.info(\"target_dir : {}\".format(target_dir+\"_\"+id_name))\n\n    # development\n    if mode:\n        normal_files = sorted(\n            glob.glob(\"{dir}/{dir_name}/{prefix_normal}_{id_name}*.{ext}\".format(dir=target_dir,\n                                                                                 dir_name=dir_name,\n                                                                                 prefix_normal=prefix_normal,\n                                                                                 id_name=id_name,\n                                                                                 ext=ext)))\n        normal_labels = numpy.zeros(len(normal_files))\n        anomaly_files = sorted(\n            glob.glob(\"{dir}/{dir_name}/{prefix_anomaly}_{id_name}*.{ext}\".format(dir=target_dir,\n                                                                                  dir_name=dir_name,\n                                                                                  prefix_anomaly=prefix_anomaly,\n                                                                                  id_name=id_name,\n                                                                                  ext=ext)))\n        anomaly_labels = numpy.ones(len(anomaly_files))\n        files = numpy.concatenate((normal_files, anomaly_files), axis=0)\n        labels = numpy.concatenate((normal_labels, anomaly_labels), axis=0)\n        com.logger.info(\"test_file  num : {num}\".format(num=len(files)))\n        if len(files) == 0:\n            com.logger.exception(\"no_wav_file!!\")\n        print(\"\\n========================================\")\n\n    # evaluation\n    else:\n        files = sorted(\n            glob.glob(\"{dir}/{dir_name}/*{id_name}*.{ext}\".format(dir=target_dir,\n                                                                  dir_name=dir_name,\n                                                                  id_name=id_name,\n                                                                  ext=ext)))\n        labels = None\n        com.logger.info(\"test_file  num : {num}\".format(num=len(files)))\n        if len(files) == 0:\n            com.logger.exception(\"no_wav_file!!\")\n        print(\"\\n=========================================\")\n\n    return files, labels","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check mode\n# \"development\": mode == True\n# \"evaluation\": mode == False\nmode, target = True, \"slider\"\nif mode is None:\n    sys.exit(-1)\n\n# make output result directory\nos.makedirs(param[\"result_directory\"], exist_ok=True)\n\n# load base directory\ndirs = com.select_dirs(param=param, mode=mode, target=target)\n\n# initialize lines in csv for AUC and pAUC\ncsv_lines = []\n\n# loop of the base directory (machine type)\nfor idx, target_dir in enumerate(dirs):\n    print(\"\\n===========================\")\n    print(\"[{idx}/{total}] {dirname}\".format(dirname=target_dir, idx=idx+1, total=len(dirs)))\n    machine_type = os.path.split(target_dir)[1]\n\n    print(\"============== MODEL LOAD ==============\")\n    # set model path\n#     model_file = \"{model}/model_{machine_type}.hdf5\".format(model=param[\"model_directory\"],\n#                                                             machine_type=machine_type)\n    model_file = \"./model/bestmodel_slider_13.hdf5\"\n\n    features_file_path = \"{features}/{machine_type}/{tip}\".format(features=param[\"features_directory\"],\n                                                                    machine_type=machine_type, tip=\"test\")\n    features_dir_path = os.path.abspath(features_file_path)\n\n    #load scaler\n    scaler_file_path = \"{scalers}/{machine_type}\".format(scalers=param[\"scalers_directory\"], machine_type=machine_type)\n    scaler_file_path = os.path.abspath(scaler_file_path)\n    scaler = load(scaler_file_path+\"/scaler_{machine_type}.bin\".format(machine_type=machine_type))\n\n\n    # load model file\n    if not os.path.exists(model_file):\n        com.logger.error(\"{} model not found \".format(machine_type))\n        #sys.exit(-1)\n        continue\n    model = load_model(model_file)\n    model.summary()\n\n\n    if mode:\n        # results by type\n        csv_lines.append([machine_type])\n        csv_lines.append([\"id\", \"AUC\", \"pAUC\"])\n        performance = []\n\n    machine_id_list = get_machine_id_list_for_test(target_dir)\n\n    # loop of the machine type directory (machine id)\n    for id_str in machine_id_list:\n        # load test file\n        test_files, y_true = test_file_list_generator(target_dir, id_str)\n\n        # setup anomaly score file path\n        anomaly_score_csv = \"{result}/anomaly_score_{machine_type}_{id_str}.csv\".format(\n                                                                                 result=param[\"result_directory\"],\n                                                                                 machine_type=machine_type,\n                                                                                 id_str=id_str)\n        anomaly_score_list = []\n\n        print(\"\\n============== BEGIN TEST FOR A MACHINE ID ==============\")\n        y_pred = [0. for k in test_files]\n        for file_idx, file_path in tqdm(enumerate(test_files), total=len(test_files)):\n\n            try:\n                # get audio features\n                vector_array = com.file_to_vector_array(file_path, None, scaler,\n                                                n_mels=param[\"feature\"][\"n_mels\"],\n                                                frames=param[\"feature\"][\"frames\"],\n                                                n_fft=param[\"feature\"][\"n_fft\"],\n                                                hop_length=param[\"feature\"][\"hop_length\"],\n                                                power=param[\"feature\"][\"power\"])\n\n                length, _ = vector_array.shape\n\n                dim = param[\"autoencoder\"][\"shape0\"]\n                step = param[\"step\"]\n\n                idex = numpy.arange(length-dim+step, step=step)\n\n                for idx in range(len(idex)):\n                    start = min(idex[idx], length - dim)\n\n                    vector = vector_array[start:start+dim,:]\n\n                    vector = vector.reshape((1, vector.shape[0], vector.shape[1]))\n                    if idx==0:\n                        batch = vector\n                    else:\n                        batch = numpy.concatenate((batch, vector))\n\n\n                # add channels dimension\n                data = batch.reshape((batch.shape[0], batch.shape[1], batch.shape[2], 1))\n\n                # calculate predictions\n                errors = numpy.mean(numpy.square(data - model.predict(data)), axis=-1)\n\n                y_pred[file_idx] = numpy.mean(errors)\n                anomaly_score_list.append([os.path.basename(file_path), y_pred[file_idx]])\n\n\n            except:\n                com.logger.error(\"file broken!!: {}\".format(file_path))\n                sys.exit(-1)\n\n        # save anomaly score\n        save_csv(save_file_path=anomaly_score_csv, save_data=anomaly_score_list)\n        com.logger.info(\"anomaly score result ->  {}\".format(anomaly_score_csv))\n\n        if mode:\n            # append AUC and pAUC to lists\n            auc = metrics.roc_auc_score(y_true, y_pred)\n            p_auc = metrics.roc_auc_score(y_true, y_pred, max_fpr=param[\"max_fpr\"])\n            csv_lines.append([id_str.split(\"_\", 1)[1], auc, p_auc])\n            performance.append([auc, p_auc])\n            com.logger.info(\"AUC : {}\".format(auc))\n            com.logger.info(\"pAUC : {}\".format(p_auc))\n\n        print(\"\\n============ END OF TEST FOR A MACHINE ID ============\")\n\n    if mode:\n        # calculate averages for AUCs and pAUCs\n        averaged_performance = numpy.mean(numpy.array(performance, dtype=float), axis=0)\n        com.logger.info(\"average AUC : {}\".format(averaged_performance[0]))\n        com.logger.info(\"average pAUC : {}\".format(averaged_performance[1]))\n        csv_lines.append([\"Average\"] + list(averaged_performance))\n        csv_lines.append([])\n\nif mode:\n    # output results\n    if target:\n        result_path = \"{result}/{target}_{file_name}\".format(result=param[\"result_directory\"], file_name=param[\"result_file\"], target=target)\n    else:\n        result_path = \"{result}/{file_name}\".format(result=param[\"result_directory\"], file_name=param[\"result_file\"])\n    com.logger.info(\"AUC and pAUC results -> {}\".format(result_path))\n    save_csv(save_file_path=result_path, save_data=csv_lines)\n    print(\"Test process for dev set finished!\")\n\nif not mode:\n    df1 = pd.read_csv('./result/anomaly_score_slider_id_01.csv', delimiter=',', header=None)\n    df3 = pd.read_csv('./result/anomaly_score_slider_id_03.csv', delimiter=',', header=None)\n    df5 = pd.read_csv('./result/anomaly_score_slider_id_05.csv', delimiter=',', header=None)\n\n    dfs = [df1, df3, df5]\n    eval_df = pd.concat(dfs)\n    eval_df = eval_df.rename(columns={0: \"file_name\", 1: \"anomaly_score\"})\n\n    eval_df.to_csv('./result/baseline_submission.csv', index=False)\n    print(\"Test process for eval set finished!\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}