{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Required Python Packages\n\nLet's install some python packages used in this notebook.","metadata":{}},{"cell_type":"markdown","source":"To import our own functions, you could upload the .py files in the input folder and do as we do for the \"common.py\" and \"pytorch_common.py\" files.","metadata":{}},{"cell_type":"code","source":"from shutil import copyfile\n\n# public modules\nimport os\nimport sklearn\nimport torch\nimport torch.nn.functional as F\nimport random\nimport librosa\nimport os\nimport glob\nimport csv\nimport re\nimport itertools\nimport sys\nimport path\nimport numpy as np\nimport seaborn as sns\nfrom torch import nn\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nfrom tqdm import tqdm\nfrom sklearn import metrics\nfrom sklearn.metrics import roc_auc_score","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-05-25T14:56:41.479417Z","iopub.execute_input":"2022-05-25T14:56:41.480453Z","iopub.status.idle":"2022-05-25T14:56:41.488753Z","shell.execute_reply.started":"2022-05-25T14:56:41.480408Z","shell.execute_reply":"2022-05-25T14:56:41.488025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def time_mask(src_melspec, T, mT, p):\n    melspec = np.copy(src_melspec)\n    num_time_steps = melspec.shape[1] \n    for i in range(mT):\n        t = 9999\n        while(p*num_time_steps < t):\n            t = np.uint16(np.random.uniform(0, T))\n        t0 = np.uint16(np.random.uniform(1, num_time_steps - t))\n        melspec[:,t0:t0+t+1] = melspec.mean() \n    return melspec\n    \ndef frequency_mask(src_melspec, F, mF):\n    melspec = np.copy(src_melspec)\n    nmels = melspec.shape[0]\n    for i in range(mF):\n        f = np.uint16(np.random.uniform(0, F))\n        f0 = np.uint16(np.random.uniform(0, nmels - f))\n        melspec[f0:f0+f+1,:] = melspec.mean()\n    return melspec\n\ndef data_aug(src_mel_spec, policy):\n    new_mel_spec = frequency_mask(src_mel_spec,policy[\"F\"], policy[\"mF\"])\n    new_mel_spec = time_mask(new_mel_spec,policy[\"T\"] ,policy[\"mT\"],policy[\"p\"])\n    return new_mel_spec\n\ndef augment_data_set(dataset, policy, N):\n    aug_data = []\n    names = []\n    class_ = ''\n    k = 0\n    for i in tqdm(range(dataset.__len__())):\n        item = dataset.__getitem__(i)\n        aug_data.append(item[0])\n        if(item[3] == 1): class_ = 'normal_' \n        else: class_ = 'anomaly_'\n        names.append(class_ + 'id_' + str(item[2]) + str(k) + '_OG')\n        k += 1\n        for i in range(N):\n            new_mel_spec = data_aug(item[0],policy)\n            aug_data.append(new_mel_spec)\n            names.append((class_ + 'id_' + str(item[2]) + str(k) + '_aug_'+ str(i)))\n            k+=1\n    return np.array(aug_data), np.array(names)\n\ndef augment_mel_spec(melspec, policy, N):\n    new_melspecs = []\n    new_melspecs.append(melspec)\n    for i in range(N):\n            new_mel_spec = data_aug(melspec,policy)\n            new_melspecs.append(new_mel_spec)\n    return np.array(new_melspecs)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-25T14:56:41.490631Z","iopub.execute_input":"2022-05-25T14:56:41.490917Z","iopub.status.idle":"2022-05-25T14:56:41.507767Z","shell.execute_reply.started":"2022-05-25T14:56:41.490882Z","shell.execute_reply":"2022-05-25T14:56:41.506842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def file_to_matrix_vector(folder_path, file_name,n_mels=64, N_crops = 5,n_fft=1024,hop_length=512,power=2.0, crop_width = 64):\n    \n    y, sr = librosa.load(os.path.join(folder_path,file_name), sr=None)\n    mel_spectrogram = librosa.feature.melspectrogram(y=y,sr=sr,n_fft=n_fft,hop_length=hop_length,n_mels=n_mels,power=power)\n    log_mel_spectrogram = 20.0 / power * np.log10(mel_spectrogram + sys.float_info.epsilon)\n    \n    dim_y = np.uint16(n_mels)\n    dim_x = np.uint16(crop_width)    \n    matrix_vector = np.zeros((N_crops, dim_y, dim_x))\n    x = 0\n    for i in range(N_crops):\n        if(x+dim_x > log_mel_spectrogram.shape[1]):\n                    x -= x+dim_x - log_mel_spectrogram.shape[1]\n        matrix_vector[i, :, :] = log_mel_spectrogram[:,x:x+dim_x]\n        x += dim_x\n    return matrix_vector\n\ndef aug_log_mel_spec_to_matrix_vector(log_mel_spectrogram,n_mels=64,N_crops=5, crop_width = 64):\n    '''\n    convert a log_mel_spectrogram (array) to a vector(array)\n    '''\n    dim_y = np.uint16(n_mels)\n    dim_x = np.uint16(crop_width)\n    matrix_vector = np.zeros((N_crops, dim_y, dim_x))\n    x = 0\n    for i in range(N_crops):\n        if(x+dim_x > log_mel_spectrogram.shape[1]):\n                    x -= x+dim_x - log_mel_spectrogram.shape[1]\n        matrix_vector[i, :, :] = log_mel_spectrogram[:,x:x+dim_x]\n        x += dim_x\n    return matrix_vector","metadata":{"execution":{"iopub.status.busy":"2022-05-25T14:56:41.509080Z","iopub.execute_input":"2022-05-25T14:56:41.509576Z","iopub.status.idle":"2022-05-25T14:56:41.522998Z","shell.execute_reply.started":"2022-05-25T14:56:41.509540Z","shell.execute_reply":"2022-05-25T14:56:41.522316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_sg_to_array_Conv(folder, data_aug = {'augment' : False, 'policy' : None, 'N' : None} ,\n                       n_mels=128, n_fft=1024, hop_length=512, power=2.0\n                       ,N_crops=5, crop_width = 32):\n    spectrograms = []\n    i = 0\n    filenames = os.listdir(folder)\n    if data_aug['augment'] :\n        for filename in tqdm(filenames):\n            y, sr = librosa.load(os.path.join(folder,filename), sr=None)\n            mel_spec = librosa.feature.melspectrogram(y=y,sr=sr,n_fft=n_fft,hop_length=hop_length,n_mels=n_mels,power=power)\n            log_mel_spectrogram = 20.0 / power * np.log10(mel_spec + sys.float_info.epsilon)\n            \n            aug_lms = augment_mel_spec(log_mel_spectrogram, data_aug['policy'], data_aug['N'])\n            \n            for aug_spec in (aug_lms):\n                spec = aug_log_mel_spec_to_matrix_vector(aug_spec, n_mels, N_crops, crop_width)\n                spectrograms[i:i+ len(spec)] = spec[:]\n                i += len(spec)    \n    else :\n        for filename in tqdm(filenames):\n            spec = file_to_matrix_vector(folder,filename,n_mels,N_crops,n_fft,hop_length,power, crop_width)\n            spectrograms[i:i + len(spec)] = spec[:]\n            i += len(spec)\n    return np.array(spectrograms)","metadata":{"execution":{"iopub.status.busy":"2022-05-25T14:56:41.526027Z","iopub.execute_input":"2022-05-25T14:56:41.526282Z","iopub.status.idle":"2022-05-25T14:56:41.537781Z","shell.execute_reply.started":"2022-05-25T14:56:41.526250Z","shell.execute_reply":"2022-05-25T14:56:41.536895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**BASELINE VAR**","metadata":{}},{"cell_type":"code","source":"from datetime import datetime\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\n\nimport keras\nfrom sklearn.preprocessing import MinMaxScaler\n\nimport tensorflow as tf\ntf.compat.v1.disable_eager_execution()\nfrom keras.models import Model, load_model\nfrom keras.layers import Lambda, Input, Dense, Flatten, BatchNormalization, Activation, Reshape\nfrom keras.layers import Conv2D, Cropping2D, Conv2DTranspose\nfrom keras.losses import mse, binary_crossentropy, kl_divergence\nfrom tensorflow.keras import optimizers\nfrom keras import backend as K\nfrom keras import initializers\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import MinMaxScaler, StandardScaler, PowerTransformer\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-05-25T14:56:41.539807Z","iopub.execute_input":"2022-05-25T14:56:41.540332Z","iopub.status.idle":"2022-05-25T14:56:41.550361Z","shell.execute_reply.started":"2022-05-25T14:56:41.540296Z","shell.execute_reply":"2022-05-25T14:56:41.549580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"A = {\"W\" : 20, \"F\" : 27, \"mF\" : 1, \"T\" : 100, \"p\" : 1.0, \"mT\" : 1}\nB = {\"W\" : 20, \"F\" : 27, \"mF\" : 2, \"T\" : 100, \"p\" : 1.0, \"mT\" : 2}\nC = {\"W\" : 10, \"F\" : 15, \"mF\" : 2, \"T\" : 70, \"p\" : 0.2, \"mT\" : 2}\nD = {\"W\" : 10, \"F\" : 27, \"mF\" : 2, \"T\" : 70, \"p\" : 1.0, \"mT\" : 2}\nE = {\"W\" : 10, \"F\" : 15, \"mF\" : 1, \"T\" : 70, \"p\" : 1.0, \"mT\" : 1}\n\nparams_conv = {'latent_dim' : 20}\n\ndata_augmentation = {'augment' : True, 'policy' : E, 'N' : 1}\nms_prms = {\"n_mels\" : 256,\"n_fft\" : 1023,\"hop_length\" : 512,\"power\" : 2.0}\n\nN_crops = 9\ncrop_width = 32\n\ntrain_dir = '../input/eurecom-aml-2022-challenge-2/eval_data/eval_data/slider/train'\ntest_dir = '../input/eurecom-aml-2022-challenge-2/dev_data/dev_data/slider/test'","metadata":{"execution":{"iopub.status.busy":"2022-05-25T14:56:41.615956Z","iopub.execute_input":"2022-05-25T14:56:41.616152Z","iopub.status.idle":"2022-05-25T14:56:41.623363Z","shell.execute_reply.started":"2022-05-25T14:56:41.616128Z","shell.execute_reply":"2022-05-25T14:56:41.622486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train  = load_sg_to_array_Conv(train_dir, data_aug = {'augment' : False, 'policy' : E, 'N' : 1},\n                                 n_mels=ms_prms['n_mels'], n_fft=ms_prms['n_fft'], hop_length=ms_prms['hop_length'], power=ms_prms['power'],\n                                 N_crops=N_crops, crop_width=crop_width)\n# X_aug_train = load_sg_to_array_Conv(train_dir, data_aug = data_augmentation,\n#                                 n_mels=ms_prms['n_mels'], n_fft=ms_prms['n_fft'], hop_length=ms_prms['hop_length'], power=ms_prms['power'], \n#                                 N_crops=N_crops, crop_width=crop_width)\nX_test  = load_sg_to_array_Conv(test_dir, data_aug = {'augment' : False, 'policy' : E, 'N' : 1},\n                                 n_mels=ms_prms['n_mels'], n_fft=ms_prms['n_fft'], hop_length=ms_prms['hop_length'], power=ms_prms['power'], \n                                 N_crops=N_crops, crop_width=crop_width)\n\n\nx_tst = X_test[:,:,:,None]\nx_tr = X_train[:,:,:,None]\n#x_aug_tr = np.copy(X_aug_train)[:,:,:,None]","metadata":{"execution":{"iopub.status.busy":"2022-05-25T14:56:41.625100Z","iopub.execute_input":"2022-05-25T14:56:41.625804Z","iopub.status.idle":"2022-05-25T14:58:28.185661Z","shell.execute_reply.started":"2022-05-25T14:56:41.625767Z","shell.execute_reply":"2022-05-25T14:58:28.184811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_file_list = os.listdir(test_dir)\ny_test_crops = np.empty(x_tst.shape[0])\ny_test = np.empty(len(test_file_list))\n\nc = 0\nfor idx, file in tqdm(enumerate(test_file_list)):\n    if file.find(\"normal\") == -1:\n        y_test[idx] = True\n        y_test_crops[c:c+N_crops] = True\n    else: \n        y_test[idx] = False\n        y_test_crops[c:c+N_crops] = False\n    c+=N_crops","metadata":{"execution":{"iopub.status.busy":"2022-05-25T14:58:28.186967Z","iopub.execute_input":"2022-05-25T14:58:28.187436Z","iopub.status.idle":"2022-05-25T14:58:28.203550Z","shell.execute_reply.started":"2022-05-25T14:58:28.187388Z","shell.execute_reply":"2022-05-25T14:58:28.202797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def combine_predicts(crop_predictions, mode='mean'):\n    c = 0\n    p = 0\n    predictions = []\n    while c <= len(crop_predictions)-5:\n        pred = crop_predictions[c:c+5]\n        if mode == 'min':\n            predictions.append(np.min(pred))\n        elif mode == 'max':\n            predictions.append(np.max(pred))\n        elif mode == 'mean':\n            predictions.append(np.mean(pred))\n        else:\n            raise ValueError('Unknown mode')\n        p+=1\n        c+=N_crops\n    return np.array(predictions)","metadata":{"execution":{"iopub.status.busy":"2022-05-25T14:58:28.205081Z","iopub.execute_input":"2022-05-25T14:58:28.205663Z","iopub.status.idle":"2022-05-25T14:58:28.213420Z","shell.execute_reply.started":"2022-05-25T14:58:28.205625Z","shell.execute_reply":"2022-05-25T14:58:28.212525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"original_dim = X_train.shape[1:3]\ninput_shape = (original_dim,)\nlatent_dim = params_conv['latent_dim']","metadata":{"execution":{"iopub.status.busy":"2022-05-25T14:58:40.840235Z","iopub.execute_input":"2022-05-25T14:58:40.840491Z","iopub.status.idle":"2022-05-25T14:58:40.845327Z","shell.execute_reply.started":"2022-05-25T14:58:40.840463Z","shell.execute_reply":"2022-05-25T14:58:40.844369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_img = Input(shape=(original_dim[0], original_dim[1], 1))\ninitializer = tf.keras.initializers.GlorotNormal()\n#initializer = tf.keras.initializers.Zeros()\nx = Conv2D(32, (5, 5),strides=(2,1), padding='same',kernel_initializer=initializer)(input_img)   #256x64 -> 128x64\nx = BatchNormalization()(x)\nx = Activation('relu')(x)\nx = Conv2D(64, (5, 5),strides=(2,2), padding='same',kernel_initializer=initializer)(x)           #64x64 Changed stride\nx = BatchNormalization()(x)\nx = Activation('relu')(x)\nx = Conv2D(128, (5, 5),strides=(2,1), padding='same',kernel_initializer=initializer)(x)          #32x32\nx = BatchNormalization()(x)\nx = Activation('relu')(x)\nx = Conv2D(256, (3, 3),strides=(2,2), padding='same',kernel_initializer=initializer)(x)          #16x16\nx = BatchNormalization()(x)\nx = Activation('relu')(x)\nx = Conv2D(512, (3, 3),strides=(2,2), padding='same',kernel_initializer=initializer)(x)          #8x8\nx = BatchNormalization()(x)\nx = Activation('relu')(x)\n\n#temp_model = Model(inputs=input_img, outputs=x)\n#temp_model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T14:58:53.319309Z","iopub.execute_input":"2022-05-25T14:58:53.319563Z","iopub.status.idle":"2022-05-25T14:58:53.619551Z","shell.execute_reply.started":"2022-05-25T14:58:53.319535Z","shell.execute_reply":"2022-05-25T14:58:53.618864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"volumeSize = K.int_shape(x)\n\nx = Conv2D(latent_dim, (4,4), strides=(1,1), padding='valid')(x)\nencoded = Flatten()(x)\n# z_mean = Dense(latent_dim)(encoded)\n# z_log_var = Dense(latent_dim)(encoded)\n# z = Lambda(sample, output_shape=(latent_dim,), name='z')([z_mean, z_log_var])","metadata":{"execution":{"iopub.status.busy":"2022-05-25T14:58:53.824149Z","iopub.execute_input":"2022-05-25T14:58:53.824392Z","iopub.status.idle":"2022-05-25T14:58:53.848090Z","shell.execute_reply.started":"2022-05-25T14:58:53.824365Z","shell.execute_reply":"2022-05-25T14:58:53.847463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = Dense(volumeSize[1] * volumeSize[2] * volumeSize[3])(encoded) \nx = Reshape((volumeSize[1], volumeSize[2], 512))(x)                #8x8\n\nx = Conv2DTranspose(256, (3, 3),strides=(2,2), padding='same',kernel_initializer=initializer)(x)  #16x16\nx = BatchNormalization()(x)\nx = Activation('relu')(x)\nx = Conv2DTranspose(128, (3, 3),strides=(2,2), padding='same',kernel_initializer=initializer)(x)  #32x32   \nx = BatchNormalization()(x)\nx = Activation('relu')(x)\nx = Conv2DTranspose(64, (5, 5),strides=(2,2), padding='same',kernel_initializer=initializer)(x)   #64x64\nx = BatchNormalization()(x)\nx = Activation('relu')(x)\nx = Conv2DTranspose(32, (5, 5),strides=(2,1), padding='same',kernel_initializer=initializer)(x)   #128x64\nx = BatchNormalization()(x)\nx = Activation('relu')(x)\nx = Conv2DTranspose(16, (5, 5),strides=(2,1), padding='same',kernel_initializer=initializer)(x)   #256x64\nx = BatchNormalization()(x)\nx = Activation('relu')(x)\n\ndecoded = Conv2DTranspose(1, (5, 5), padding='same')(x)\nconv_AE = Model(inputs=input_img, outputs=decoded)\n# conv_AE.summary()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T14:58:54.262830Z","iopub.execute_input":"2022-05-25T14:58:54.263053Z","iopub.status.idle":"2022-05-25T14:58:54.614503Z","shell.execute_reply.started":"2022-05-25T14:58:54.263027Z","shell.execute_reply":"2022-05-25T14:58:54.613777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"opt = optimizers.Adam(learning_rate=0.001, clipvalue=0.5)\n#opt = optimizers.RMSprop(learning_rate=0.0001)\ntop_score = 0\nweight_best_model = None\nconv_AE.compile(optimizer=opt, loss=\"mse\")\nconv_AE.summary()\n# Finally, we train the model:","metadata":{"execution":{"iopub.status.busy":"2022-05-25T14:58:54.809835Z","iopub.execute_input":"2022-05-25T14:58:54.810085Z","iopub.status.idle":"2022-05-25T14:58:54.851769Z","shell.execute_reply.started":"2022-05-25T14:58:54.810057Z","shell.execute_reply":"2022-05-25T14:58:54.851026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomCallback(keras.callbacks.Callback):\n    def on_epoch_end(self, epoch, logs=None):\n        X_test_pred = conv_AE.predict(x_tst)[:,:,:,0]\n        x_tst_3 = x_tst[:,:,:,0]\n        mae_vector_test = np.zeros(X_test_pred.shape[0])\n        for idx, x_pred in tqdm(enumerate(X_test_pred)):\n            mae_vector_test[idx] = np.mean((np.absolute((x_pred-x_tst_3[idx]))))\n        y_test_predict = combine_predicts(mae_vector_test, mode='mean')\n        auc_val_score = roc_auc_score(y_test, y_test_predict)\n        print(auc_val_score)\n        if auc_val_score > top_score:\n            weight_best_model = conv_AE.get_weights()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T14:58:55.425892Z","iopub.execute_input":"2022-05-25T14:58:55.426413Z","iopub.status.idle":"2022-05-25T14:58:55.433779Z","shell.execute_reply.started":"2022-05-25T14:58:55.426378Z","shell.execute_reply":"2022-05-25T14:58:55.433091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = conv_AE.fit(x_tr, x_tr,shuffle=True,epochs=20, batch_size=254, callbacks=[CustomCallback()])","metadata":{"execution":{"iopub.status.busy":"2022-05-25T14:58:56.228205Z","iopub.execute_input":"2022-05-25T14:58:56.228748Z","iopub.status.idle":"2022-05-25T14:59:31.504942Z","shell.execute_reply.started":"2022-05-25T14:58:56.228711Z","shell.execute_reply":"2022-05-25T14:59:31.503975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#conv_AE.set_weights(weight_best_model) \nX_test_pred = conv_AE.predict(x_tst)[:,:,:,0]\nx_tst_3 = x_tst[:,:,:,0]\nmae_vector_test = np.zeros(X_test_pred.shape[0])\nfor idx, x_pred in tqdm(enumerate(X_test_pred)):\n    mae_vector_test[idx] = np.mean((np.absolute((x_pred-x_tst_3[idx]))))","metadata":{"execution":{"iopub.status.busy":"2022-05-25T14:59:34.893321Z","iopub.execute_input":"2022-05-25T14:59:34.894370Z","iopub.status.idle":"2022-05-25T14:59:39.502135Z","shell.execute_reply.started":"2022-05-25T14:59:34.894322Z","shell.execute_reply":"2022-05-25T14:59:39.501303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test_predict = combine_predicts(mae_vector_test, mode='mean')","metadata":{"execution":{"iopub.status.busy":"2022-05-25T14:59:40.862421Z","iopub.execute_input":"2022-05-25T14:59:40.863228Z","iopub.status.idle":"2022-05-25T14:59:40.879688Z","shell.execute_reply.started":"2022-05-25T14:59:40.863180Z","shell.execute_reply":"2022-05-25T14:59:40.878949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"roc_auc_score(y_test, y_test_predict)","metadata":{"execution":{"iopub.status.busy":"2022-05-25T14:59:41.655055Z","iopub.execute_input":"2022-05-25T14:59:41.655593Z","iopub.status.idle":"2022-05-25T14:59:41.669953Z","shell.execute_reply.started":"2022-05-25T14:59:41.655545Z","shell.execute_reply":"2022-05-25T14:59:41.669164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"import gc\ngc.collect()\ndel X_train, x_tr\nsubmit_dir = '../input/eurecom-aml-2022-challenge-2/eval_data/eval_data/slider/test'\nX_submit  = load_sg_to_array_Conv(submit_dir, data_aug = {'augment' : False, 'policy' : E, 'N' : 1},\n                             n_mels=ms_prms['n_mels'], n_fft=ms_prms['n_fft'], hop_length=ms_prms['hop_length'], power=ms_prms['power'], \n                             N_crops=N_crops, crop_width=crop_width)\nx_sbmt = np.copy(X_submit)[:,:,:,None]\nx_sbmt_3 = x_sbmt[:,:,:,0]\nX_submit_pred = conv_AE.predict(x_sbmt)[:,:,:,0]\nmae_vector_submit = np.zeros(X_submit_pred.shape[0])\nfor idx, x_pred in tqdm(enumerate(X_submit_pred)):\n    mae_vector_submit[idx] = np.mean((np.absolute((x_pred-x_sbmt_3[idx]))))\ny_submit_predict = combine_predicts(mae_vector_submit, mode='max')","metadata":{"execution":{"iopub.status.busy":"2022-05-25T14:59:43.397708Z","iopub.execute_input":"2022-05-25T14:59:43.397985Z","iopub.status.idle":"2022-05-25T15:00:14.143008Z","shell.execute_reply.started":"2022-05-25T14:59:43.397953Z","shell.execute_reply":"2022-05-25T15:00:14.142177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nsubmit_files = os.listdir(submit_dir)\ndf_submit = pd.DataFrame({'file_name': submit_files, 'anomaly_score': y_submit_predict})\ndf_submit.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-25T15:00:14.144861Z","iopub.execute_input":"2022-05-25T15:00:14.145557Z","iopub.status.idle":"2022-05-25T15:00:14.169021Z","shell.execute_reply.started":"2022-05-25T15:00:14.145506Z","shell.execute_reply":"2022-05-25T15:00:14.168302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}