{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **AML Challenge 2 - Anomalous Sound Detection - Group 14**\n##### **Carl-Johan BJÖRNSON, Håkon GRAESBERG, Christian NGUYEN, Mathias WOLD**\n\nThis notebook contains the models discussed in our presentation report and is designed to run on Google Colab. \n\nRemark the paths might need to be adjusted to the specific user.\n\n### To see output of the cells, please open the pre-run notebook in Google Colab [here](https://colab.research.google.com/drive/1G5zyIqolPu02D2LujHh1y7VAK9ESoXNe?usp=sharing)","metadata":{"id":"tvPb3KiImpZ8"}},{"cell_type":"markdown","source":"### Kaggle setup\nThe dataset for this challenge was fetched from the Kaggle competition. This requires an Kaggle API Token.","metadata":{"id":"gPEEJLCn0wM5"}},{"cell_type":"code","source":"! pip install -q kaggle","metadata":{"id":"OpDvCs140sRY"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Upload Kaggle API Token\nfrom google.colab import files\nfiles.upload()         # expire any previous token(s) and upload recreated token","metadata":{"id":"WNGVSKRV01zv"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! mkdir ~/.kaggle\n! cp kaggle.json ~/.kaggle/\n! chmod 600 ~/.kaggle/kaggle.json\n#! kaggle datasets list","metadata":{"id":"kdCaAF5A04Id"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Unzip data to sample_data directory\n! kaggle competitions download -c \"eurecom-aml-2022-challenge-2\"\n! unzip eurecom-aml-2022-challenge-2.zip -d sample_data","metadata":{"id":"5XzUgSW006jt"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The python files 'common.py' and 'pytorch_common.py', stored in shared Google Drive folder\nfrom google.colab import drive\ndrive.mount('/content/drive')","metadata":{"id":"6ZxuOIq23-VW"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Imports and Installs\n\n","metadata":{"id":"EZ9twsrTm6gt"}},{"cell_type":"code","source":"from shutil import copyfile\n\n# copy our file into the working directory (make sure it has .py suffix)\ncopyfile(src = \"./drive/MyDrive/AML/challenge2/pytorch_common.py\", dst = \"/content/pytorch_common.py\")\ncopyfile(src = \"./drive/MyDrive/AML/challenge2/common.py\", dst = \"/content/common.py\")\n\n# import all our functions\n\nimport common as com\nimport pytorch_common\nfrom pytorch_common import *\n\n# public modules\nimport cv2\nimport torch\nimport torch.nn.functional as F\nimport torchsummary\nimport pytorch_lightning as pl\nimport random\nimport librosa\nimport os\nimport glob\nimport csv\nimport re\nimport itertools\nimport sys\nimport path\nimport numpy\nimport seaborn as sns\nfrom torch import nn\nfrom tqdm import tqdm\nfrom sklearn import metrics\nfrom dlcliche.utils import *\nfrom dlcliche.notebook import *\nfrom dlcliche.utils import (\n    sys, random, Path, np, plt, EasyDict,\n    ensure_folder, deterministic_everything,\n)\n\nfrom sklearn.svm import OneClassSVM\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import PCA, IncrementalPCA\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import roc_auc_score","metadata":{"id":"tjlfInpx0g3Q","scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install torchsummary \n!pip install dl-cliche\n!pip install path\n!pip install pytorch_lightning\n!pip install torchvision\n!pip install opencv-python","metadata":{"outputId":"f369b82c-8b09-4c3a-b4b0-2509c0cb6565","id":"_GFdvW5D7nka"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torchvision\n\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\n\nfrom torchvision import datasets, models, transforms\nfrom torchvision.datasets import ImageFolder\n\nfrom torch.utils.data import DataLoader,Dataset,ConcatDataset\nfrom torchvision.utils import make_grid\n\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n\nfrom matplotlib.image import imread\n\nimport time\nimport os\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tqdm\nprint(\"PyTorch Version: \",torch.__version__)\nprint(\"Torchvision Version: \",torchvision.__version__)","metadata":{"id":"NU_jn17ypkzg","outputId":"47cf3c10-82cf-4118-8ff7-61bb5e7340fc"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The Data","metadata":{"id":"CHZaIWUm8Ljv"}},{"cell_type":"markdown","source":"### Quick data exploration","metadata":{"id":"Y8AQoV8r8iKo"}},{"cell_type":"code","source":"# development dataset\n\nDATA_ROOT=Path(\"/content/sample_data/dev_data/dev_data\")\n\ntypes = [t.name for t in sorted(DATA_ROOT.glob('*')) if t.is_dir()]\nprint('Machine types:', types)\n\ndf = pd.DataFrame()\ndf['file'] = sorted(DATA_ROOT.glob('*/*/*.wav'))\ndf['type'] = df.file.map(lambda f: f.parent.parent.name)\ndf['split'] = df.file.map(lambda f: f.parent.name)\ndf['id'] = df.file.map(lambda s: str(s).split('/')[-1].split('_')[-2])\n\nagg = df.groupby(['id', 'split']).agg('count')\nfig = plt.figure(figsize=(12.0, 6.0))\ng = sns.barplot(x=\"id\", y=\"file\", hue=\"split\", data=agg.reset_index())\nplt.xlabel(\"machine id\")\nplt.ylabel(\"number of files\")\nplt.show()\nagg.transpose()","metadata":{"id":"fN2xmw0H0g3R","outputId":"f85e68cb-bb5e-416d-a8f3-07efe48bc3c6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# evaluation dataset\nDATA_ROOT=Path(\"/content/sample_data/eval_data/eval_data\")\ntypes = [t.name for t in sorted(DATA_ROOT.glob('*')) if t.is_dir()]\nprint('Machine types:', types)\n\ndf = pd.DataFrame()\ndf['file'] = sorted(DATA_ROOT.glob('*/*/*.wav'))\ndf['type'] = df.file.map(lambda f: f.parent.parent.name)\ndf['split'] = df.file.map(lambda f: f.parent.name)\ndf['id'] = df.file.map(lambda s: str(s).split('/')[-1].split('_')[-2])\n\nagg = df.groupby(['id', 'split']).agg('count')\nfig = plt.figure(figsize=(12.0, 6.0))\ng = sns.barplot(x=\"id\", y=\"file\", hue=\"split\", data=agg.reset_index())\nplt.xlabel(\"machine id\")\nplt.ylabel(\"number of files\")\nplt.show()\nagg.transpose()","metadata":{"id":"cOUTrVHO0g3S","outputId":"aa0a3e8d-a2c1-4334-f5f0-8d202de6c150"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_log_mel_spectrogram(filename, n_mels=64,\n                        n_fft=1024,\n                        hop_length=512,\n                        power=2.0):\n    wav, sampling_rate = com.file_load(filename)\n    mel_spectrogram = librosa.feature.melspectrogram(y=wav,\n                                                     sr=sampling_rate,\n                                                     n_fft=n_fft,\n                                                     hop_length=hop_length,\n                                                     n_mels=n_mels,\n                                                     power=power)\n    log_mel_spectrogram = 20.0 / power * np.log10(mel_spectrogram + sys.float_info.epsilon)\n    return log_mel_spectrogram, wav\n\nfor t in types:\n    for split in ['train', 'test']:\n        type_df = df[df['type'] == t][df.split == split].reset_index()\n        R = 2\n        fig, ax = plt.subplots(R, 2, figsize = (15, 1.5*R))\n        print(f'=== Machine type [{t}], {split} set ===')\n        for i in range(R * 2):\n            file_index = i//2\n            file_path = Path(type_df.file[file_index])\n            mels, wav = get_log_mel_spectrogram(file_path)\n            if i%2==0:\n                ax[i//2, i%2].set_title(file_path.name)\n                ax[i//2, i%2].imshow(mels)\n                ax[i//2, i%2].axis('off')\n            else:\n                ax[i//2, i%2].set_title(file_path.name)\n                ax[i//2, i%2].plot(wav)\n                ax[i//2, i%2].get_xaxis().set_ticks([])\n        plt.show()","metadata":{"id":"sgaTu0au0g3T","outputId":"e2ff8fea-95da-4976-834a-6e2cfb9ef057"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Creating and storing Mel Spectrograms\n","metadata":{"id":"G6pVfHUZ81g9"}},{"cell_type":"code","source":"# assign directory\ndirectory=Path(\"/content/sample_data/dev_data/dev_data/slider/test\")\ndst=\"/content/mel_spect_format_as_paper\"\nos.mkdir(dst)\nos.mkdir(dst + \"/mel\")\n\n# iterate over files in\n# that directory\nfor filename in os.listdir(directory):\n    f = os.path.join(directory, filename)\n    # checking if it is a file\n    if os.path.isfile(f):\n        mel, _ = get_log_mel_spectrogram(f)\n        name = os.path.splitext(filename)[0]\n        np.save(dst +\"/mel/\"+ name, mel)","metadata":{"id":"AC-NsMHZ_Aee"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ResNet34 Feature Extraction","metadata":{"id":"0S1eV5j_71PT"}},{"cell_type":"code","source":"def get_log_mel_spectrogram(filename, n_mels=64,\n                        n_fft=1024,\n                        hop_length=256,\n                        win_length=1024,\n                        power=2.0):\n    wav, sampling_rate = com.file_load(filename)\n    mel_spectrogram = librosa.feature.melspectrogram(y=wav,\n                                                     sr=sampling_rate,\n                                                     n_fft=n_fft,\n                                                     hop_length=hop_length,\n                                                     n_mels=n_mels,\n                                                     power=power)\n    log_mel_spectrogram = 20.0 / power * np.log10(mel_spectrogram + sys.float_info.epsilon)\n    return log_mel_spectrogram, wav","metadata":{"id":"t1kQxfN2HR0s"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path='/content/sample_data/dev_data/dev_data/slider'\n\nBATCH_SIZE=32\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\ndevice","metadata":{"id":"MI6-bjgqpkv5","outputId":"68597295-f5ad-4758-cebd-3689a77b0540"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resnet34 = models.resnet34(True, True)","metadata":{"id":"am1iph_C41d0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reshape_one_img_to_rgb(img, part):\n  '''\n  Maximum num is 19. We loose the last 18 pixels ~0.56sec\n  '''\n  resized = cv2.resize(img[:,part*32:part*32+64], (224,224))\n  std = (resized - np.mean(resized)) / np.std(resized)\n  backtorgb = cv2.cvtColor(std,cv2.COLOR_GRAY2RGB)\n  #print(backtorgb.shape)  # (244, 224, 3) = (width, height, channels)\n  x = np.moveaxis(backtorgb, -1, 0)\n  #print(x.shape) # (3,244,244) = (channel, widht, height)\n  #rgb_batch = np.repeat(x[np.newaxis, ...], 3, -1)\n  #x = np.asarray(x).reshape((1, x.shape[0], x.shape[1], x.shape[2]))\n  #print(x.shape)\n  #standarized_x = (x - np.mean(x))/np.std(x)\n  return x","metadata":{"id":"xpaMvz3J4xuQ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def retrieve_all_slides(filename):\n  img_array = np.load(filename)\n  storage = []\n  for i in range(20):\n    storage.append(reshape_one_img_to_rgb(img_array, i))\n  return np.array(storage)","metadata":{"id":"YZkyfpoc4u3S"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_feature_vectors_of_one_sample(filename):\n  resnet34.eval()\n  one_sample = retrieve_all_slides(filename)\n  tensor_img = torch.as_tensor(one_sample)\n  out = resnet34(tensor_img)\n  out_np = out.detach().numpy()\n  i = 0\n  for vec in out_np:\n    out_np[i] = (vec - np.mean(vec))/np.std(vec)\n    i+=1\n    #standarized_x = (x - np.mean(x))/np.std(x)\n  return out_np","metadata":{"id":"lSP8ukQH7mzi"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_filename = '/content/mel_spect_format_as_paper/mel/normal_id_00_00000385.npy'\none_sample = get_feature_vectors_of_one_sample(test_filename)","metadata":{"id":"sW8PkoeV8NcN"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(one_sample.shape, type(one_sample))","metadata":{"id":"B7e3rZgs-O6L","outputId":"8f545f02-86c0-4d50-be13-8d5b74a8e132"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Converting Spectograms and storing feature vectors as numpy array\nConverting the mel spectograms to feature vectors is a time consuming operation. This was therefore only done once with all the data and stored as numpy arrays on google drive for all of us in the group to access. ","metadata":{"id":"sKWTCrrQ-LUU"}},{"cell_type":"code","source":"X_test = []\nX_test_names = []\ndirectory = '/content/mel_spect_format_as_paper/mel/'\nfor filename in os.listdir(directory):\n    f = os.path.join(directory, filename)\n    if os.path.isfile(f):\n      one_sample = get_feature_vectors_of_one_sample(f)\n      X_test.append(one_sample)\n      X_test_names.append(str(filename).split('.')[0] + '.wav')\n\nX_test = np.array(X_test)\nX_test_names = np.array(X_test_names)\n#y_test = np.ones(len(X_test))\n\ndst = '/content/drive/MyDrive/AML/challenge2/resnet_data/dev'\nname = 'transformed_Xtest'\nname2 = 'transformed_Xtest_names'\nnp.save(dst +\"/\"+ name, X_test)\nnp.save(dst +\"/\"+ name2, X_test_names)","metadata":{"id":"414HmbymaVZ_"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reading stored numpy arrays","metadata":{"id":"-z9MxNTf_Ho0"}},{"cell_type":"code","source":"X_train = np.load('/content/drive/MyDrive/AML/challenge2/resnet_data/eval/transformed_Xtrain.npy')\n(a,b,c) = X_train.shape\nX_train = X_train.reshape((a*b),c)\nprint(X_train.shape)\ny_train = np.ones(len(X_train))","metadata":{"id":"uqzwCioI2XXA","outputId":"c6228eb2-614e-4445-a9fd-636ae6a9d684"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read in testing data\nX_test_names = np.load('/content/drive/MyDrive/AML/challenge2/resnet_data/eval/transformed_Xtest_names.npy')\nX_test = np.load('/content/drive/MyDrive/AML/challenge2/resnet_data/eval/transformed_Xtest.npy')\n\n# Include this when testing on devalopment dataset\n'''\ny_test = []\n\nfor i in X_test_names:\n  label  = str(i).split('_')[0]\n  if label == 'anomaly':\n    y_test.append(-1)\n  elif label =='normal':\n    y_test.append(1)\n'''","metadata":{"id":"uHtLK1j0apd2","outputId":"01a7346e-aaee-4ad3-c6e1-d40b71bf4e79"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## One-Class SVM ","metadata":{"id":"fbuZM3m3AObt"}},{"cell_type":"code","source":"def NormalizeData(data):\n    return (data - np.min(data)) / (np.max(data) - np.min(data))","metadata":{"id":"NHrQIKKa2Dx-"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_anomaly_score(X, oc_svm):\n  avg_scores = []\n  for x in X:\n    # returnes the signed distance from the decision boundary, negative for outliers and positive for inliers.\n    signed_distances = oc_svm.decision_function(x)\n    # Change sign since we want it in anomaly score, where 1 is the most anomaluous and 0 the least.\n    signed_distances *= -1 \n    # Computing avg distance for the file\n    avg_dist = np.mean(signed_distances)\n    avg_scores.append(avg_dist)\n  # Normalize to get all the distances in the range 0..1\n  anomaly_scores = NormalizeData(avg_scores)\n  return anomaly_scores","metadata":{"id":"Um7sO8zpuuqH"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_predictions(X_test, oc_svm):\n  predictions = []\n  for x in X_test:\n    sample_pred = oc_svm.predict(x)\n    unique, counts = np.unique(sample_pred, return_counts=True)\n    index = np.argmax(counts)\n    majority = unique[index]\n    predictions.append(majority)\n  return predictions","metadata":{"id":"VVoHJwMs6GdU"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Optimal parameters found by Grid Search\nnu = 0.001\ngamma = 0.02\nkernel = 'rbf'\n\none_class_svm = OneClassSVM(nu=nu, kernel = kernel, gamma = gamma)\none_class_svm.fit(X_train)\npredictions_train = one_class_svm.predict(X_train)\npredictions_test = get_predictions(X_test, one_class_svm)\nunique, counts = np.unique(predictions_test, return_counts=True)\nprint('__________________________________')\nprint('nu:', nu, 'gamma:', gamma)\ntrain_report = classification_report(y_train, predictions_train)\n\ntest_report = classification_report(y_test, predictions_test)\nprint('----------Training data----------')\nprint(train_report)\nprint('----------Testing data-----------')\n#print(test_report)\n#print(unique, counts)","metadata":{"id":"Hf0Q3RTaU72q","outputId":"344668f1-20df-4c76-8446-f71a188778ee"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Total ROC:\", roc_auc_score(y_test, predictions_test))","metadata":{"id":"9sm9rVJoqFTU","outputId":"444924a7-29ad-4e17-9789-f757ab29fc41"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anomaly_score = get_anomaly_score(X_test, one_class_svm)","metadata":{"id":"TD2uVlKVA-CZ"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Make submission file ","metadata":{"id":"tuuyKXgJCaJS"}},{"cell_type":"code","source":"submission_df = pd.DataFrame()\nsubmission_df[\"file_name\"] = X_test_names.tolist()\nsubmission_df[\"anomaly_score\"] = anomaly_score\nsubmission_df.to_csv('/content/sample_data/feature_extraction_submission01.csv', index = False)","metadata":{"id":"xlWNgbtA3TMA"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"DitGhyywBjx1"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Gaussian Mixture Model (GMM) with ResNet34 for anomaly detection","metadata":{"id":"6yUkwY5-46Qx"}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.mixture import GaussianMixture\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.metrics import roc_auc_score","metadata":{"id":"1LPfeQqV4bjB"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loading training and test data","metadata":{"id":"fMd0GpAj5Suz"}},{"cell_type":"code","source":"X_train = np.load('/content/drive/MyDrive/EURECOM/AML/challenge2/resnet_data/dev/transformed_Xtrain.npy')\nX_train = X_train.reshape(len(X_train)*len(X_train[0]), 1000)\n\nX_test_names = np.load('/content/drive/MyDrive/EURECOM/AML/challenge2/resnet_data/dev/transformed_Xtest_names.npy')\nX_test = np.load('/content/drive/MyDrive/EURECOM/AML/challenge2/resnet_data/dev/transformed_Xtest.npy')\nX_test = X_test.reshape(len(X_test)*len(X_test[0]), 1000)\n\ny_test = []\n\nfor i in X_test_names:\n  label  = str(i).split('_')[0]\n  if label == 'anomaly':\n    y_test.append(1)\n  elif label =='normal':\n    y_test.append(0)\n\nprint(\"Training set\", X_train.shape)\nprint(\"Testing set\", X_test.shape)","metadata":{"id":"--AyaaF6T9XP","outputId":"ed4cb4f8-5d3c-4b71-d1d8-22dab12e37e3"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training the GMM","metadata":{"id":"1ko9HJ-o5jCT"}},{"cell_type":"code","source":"gm = GaussianMixture(n_components = 20, covariance_type = 'diag', random_state=0, max_iter=200, warm_start=True)\ngm.fit(X_train)","metadata":{"id":"JuibReY0Ryi8","outputId":"84c70649-645c-4251-a85b-9b5f95fc644d"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test and calculate anomaly score","metadata":{"id":"CHT9iJtD5osj"}},{"cell_type":"code","source":"list_of_anomaly_scores = []\ntemp_scores = []\nscores = gm.score_samples(X_test)\ncounter = 0\nfor i in range(len(X_test)):\n  temp_scores.append(scores[i].max())\n  counter+=1\n  # Average every 20 stored samples.\n  if counter==20:\n    single_anomaly_score = np.mean(np.array(temp_scores))\n    list_of_anomaly_scores.append(single_anomaly_score)\n    temp_scores = []\n    counter=0\n\n# Scaling log-probability for anomaly score\nscaler = MinMaxScaler(feature_range=(0, 1))\nscaled = scaler.fit_transform([[x] for x in list_of_anomaly_scores])\nlist_of_anomaly_scores = 1-scaled\n\nprint('\\n')\nprint('--------------All machines---------------')\nprint(\"AUC score:\", roc_auc_score(y_test, list_of_anomaly_scores))\nprint('---------------------------------------')\nprint('\\n')\n","metadata":{"id":"7nwODH4255u1","outputId":"3a6d7adf-f01f-4ae9-9cab-0572dc4815b8"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Make submission file","metadata":{"id":"K-YCs7T-59MB"}},{"cell_type":"code","source":"submission_df = pd.DataFrame()\nsubmission_df[\"file_name\"] = X_test_names.tolist()\nsubmission_df[\"anomaly_score\"] = list_of_anomaly_scores\nsubmission_df.to_csv('/content/sample_data/feature_extraction_submission.csv', index = False)","metadata":{"id":"OrRjxJc7H7xq"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Convolutional Autoencoder","metadata":{"id":"OHVSVr1VCrtG"}},{"cell_type":"code","source":"! pip install pytorch-lightning typing_extensions","metadata":{"id":"n8ui8nWRc3H6","outputId":"391e11c0-a379-4e6a-948d-a0edac9a089f"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nfrom matplotlib import rc\n\nrc(\"font\", **{\"family\": \"sans-serif\", \"sans-serif\": \"DejaVu Sans\"})\nrc(\"figure\", **{\"dpi\": 200})\nrc(\n    \"axes\",\n    **{\"spines.right\": False, \"spines.top\": False, \"xmargin\": 0.0, \"ymargin\": 0.05}\n)","metadata":{"id":"t7wao5eT2cMP"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data","metadata":{"id":"kMSzbpId3h45"}},{"cell_type":"code","source":"import sys\nimport librosa\n\ndef get_log_mel_spectrogram(filename, n_mels=128, n_fft=1024, hop_length=512, power=1.0):\n    wav, sampling_rate = librosa.load(filename, sr=None, mono=False)\n    mel_spectrogram = librosa.feature.melspectrogram(y=wav,\n                                                     sr=sampling_rate,\n                                                     n_fft=n_fft,\n                                                     hop_length=hop_length,\n                                                     n_mels=n_mels,\n                                                     power=power)\n    log_mel_spectrogram = 20.0 / power * np.log10(mel_spectrogram + sys.float_info.epsilon)\n    return log_mel_spectrogram, wav","metadata":{"id":"HsrfPPHOzZKf"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's plot a waveform and it's corresponding log mel spectrogram:","metadata":{"id":"d3lk5Djn5UsD"}},{"cell_type":"code","source":"filename = \"dev_data/dev_data/slider/test/anomaly_id_00_00000000.wav\"\n\nlog_mel_spectrogram, wav = get_log_mel_spectrogram(filename)\n\n# plot wav\nfig, ax = plt.subplots(1, 1, figsize=(6, 3))\nax.set_title(f\"Waveform of {filename.split('/')[-1]}\")\nax.plot(wav)\nax.axis('off')\nplt.show()","metadata":{"id":"_tyCVfJAZJTJ","outputId":"be900d49-082d-4681-a740-1b73d269f7ce"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot log mel\nfig, ax = plt.subplots(1, 1, figsize=(6, 3))\nax.set_title(f\"Log Mel Spectrogram of {filename.split('/')[-1]}\")\nax.imshow(log_mel_spectrogram)\nax.axis('off')\nplt.show()","metadata":{"id":"wlAM5D2jbU4d","outputId":"a0634954-4e2b-48f9-9b5d-2c6e06a59e29"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We now create a pytorch dataset class for storing the spectrograms:","metadata":{"id":"W4hNA7XhQ-Z7"}},{"cell_type":"code","source":"import os\nfrom tqdm import tqdm\nimport torch\nfrom sklearn.preprocessing import StandardScaler\nfrom pickle import dump, load\n\nclass SpectrogramDataset(torch.utils.data.Dataset):\n    def __init__(self, directory):\n        main_dataset_type = \"dev\" if \"dev\" in directory else \"eval\"\n        dataset_type  = \"train\" if \"train\" in directory else \"test\"\n\n        spectogram_dir = f\"mel_spectrograms_{main_dataset_type}_{dataset_type}\"\n\n        os.makedirs(spectogram_dir, exist_ok=True)\n        os.makedirs(\"scalers\", exist_ok=True)\n\n        scaler_path = f\"scalers/{main_dataset_type}_{dataset_type}.pkl\"\n\n        # cache the spectrograms and the standard scaler if cache does not exist\n        if not len(os.listdir(spectogram_dir)) == len(os.listdir(directory)):\n            scaler = StandardScaler()\n            for filename in tqdm(os.listdir(directory), desc=\"Creating spectrograms\"):\n                name = os.path.splitext(filename)[0]\n                spectrogram_path = f\"{spectogram_dir}/{name}\"\n                path = os.path.join(directory, filename)\n                spectogram, _ = get_log_mel_spectrogram(path)\n                scaler.partial_fit(spectogram)\n                np.save(spectrogram_path, spectogram)\n            dump(scaler, open(scaler_path, \"wb\"))\n            \n        \n        self.spectrograms = []\n        self.labels = []\n        self.filenames = []\n\n        # use the cached spectrograms and standard scaler\n        scaler = load(open(scaler_path, \"rb\"))\n        for filename in os.listdir(spectogram_dir):\n            spectrogram = np.load(f\"{spectogram_dir}/{filename}\")\n            spectrogram = scaler.transform(spectrogram)\n            # cut some rows and columns to make cnn dimensions happy\n            self.spectrograms.append(spectrogram[:121,:301])\n            self.labels.append(1 if filename.split(\"_\")[0] == \"anomaly\" else 0)\n            self.filenames.append(filename.split(\".\")[0])\n\n    def __len__(self):\n        return len(self.spectrograms)\n\n    def __getitem__(self, index):\n        return self.spectrograms[index], self.labels[index], self.filenames[index]","metadata":{"id":"SLUY2sVu6nGk"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# uncomment to remove cache\n#! rm -rf mel_spectrograms_dev_train/\n#! rm -rf mel_spectrograms_dev_test/\n#! rm -rf mel_spectrograms_eval_train/\n#! rm -rf mel_spectrograms_eval_test/\n#! rm -rf scalers/","metadata":{"id":"ljCnufoO5kQm"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# populate the cache directories\nSpectrogramDataset(\"dev_data/dev_data/slider/train\")\nSpectrogramDataset(\"dev_data/dev_data/slider/test\")\nSpectrogramDataset(\"eval_data/eval_data/slider/train\")\nSpectrogramDataset(\"eval_data/eval_data/slider/test\")","metadata":{"id":"a7ardBaN5qtB","outputId":"a8ed779f-3ca2-4fd3-872c-053fc1b16d3e"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model","metadata":{"id":"gc7WmlNQ7SkI"}},{"cell_type":"code","source":"from torch.nn.modules.conv import ConvTranspose2d\nfrom torch import nn\nimport torch.nn.functional as F\nimport pytorch_lightning as pl\n\nclass CNNAutoEncoder(pl.LightningModule):\n    def __init__(self):\n        super().__init__()\n        self.loss_func = nn.MSELoss()\n\n        self.encoder = nn.Sequential(\n            nn.Conv2d(1, 32, 5, stride=(1,2)),\n            nn.BatchNorm2d(32),\n            nn.ReLU(),\n            nn.Conv2d(32, 64, 5, stride=(1,2)),\n            nn.BatchNorm2d(64),\n            nn.ReLU(),\n            nn.Conv2d(64, 128, 5, stride=(2,2)),\n            nn.BatchNorm2d(128),\n            nn.ReLU(),\n            nn.Conv2d(128, 256, 3, stride=(2,2)),\n            nn.BatchNorm2d(256),\n            nn.ReLU(),\n            nn.Conv2d(256, 512, 3, stride=(2,2)),\n            nn.BatchNorm2d(512),\n            nn.ReLU(),\n        )\n        self.flatten = nn.Sequential(\n            nn.Conv2d(512, 40, 4, stride=(1,1)),\n            nn.Flatten(),\n        )\n        self.decoder = nn.Sequential(\n            nn.Linear(2000, 512 * 13 * 8),\n            nn.Unflatten(dim=1, unflattened_size=(512, 13, 8)),\n            ConvTranspose2d(512, 256, 3, stride=(2,2)),\n            nn.BatchNorm2d(256),\n            nn.ReLU(),\n            ConvTranspose2d(256, 128, 3, stride=(2,2)),\n            nn.BatchNorm2d(128),\n            nn.ReLU(),\n            ConvTranspose2d(128, 64, 5, stride=(2,2)),\n            nn.BatchNorm2d(64),\n            nn.ReLU(),\n            ConvTranspose2d(64, 32, 5, stride=(1,2)),\n            nn.BatchNorm2d(32),\n            nn.ReLU(),\n            ConvTranspose2d(32, 1, 5, stride=(1,2)),\n        )\n \n\n    def forward(self, x):\n        x = self.encoder(x)\n        x = self.flatten(x)\n        x = self.decoder(x)\n        return x\n\n    def training_step(self, batch, batch_idx):\n        x, _, _ = batch\n        # add a dimension for number of channels\n        # from (batch_size, height, width) to (batch_size, n_channels, height, width)\n        # where n_channels = 1\n        x = x.unsqueeze(1)\n        x_hat = self.forward(x)\n        loss = self.loss_func(x_hat, x)\n        return loss\n\n    def configure_optimizers(self):\n        return torch.optim.Adam(self.parameters(), lr=1e-3, weight_decay=1e-05)\n        ","metadata":{"id":"SaHYZmDF394S"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training on dev data","metadata":{"id":"Di0oDVlS5S49"}},{"cell_type":"code","source":"from typing_extensions import Literal\n\ndef train(dataset_type: Literal[\"dev\", \"eval\"]):\n    assert dataset_type == \"dev\" or dataset_type == \"eval\"\n    os.makedirs(\"models\", exist_ok=True)\n    trainer = pl.Trainer(\n        accelerator=\"gpu\",\n        gpus=torch.cuda.device_count(),\n        max_epochs=100,\n    )\n    model = CNNAutoEncoder()\n    dataset = SpectrogramDataset(f\"{dataset_type}_data/{dataset_type}_data/slider/train\")\n    dataloader = torch.utils.data.DataLoader(dataset, batch_size=64, shuffle=True)\n    trainer.fit(model=model, train_dataloaders=dataloader)\n    torch.save(model.state_dict(), f\"models/{dataset_type}.pt\")","metadata":{"id":"jbFuYgZdHH-G"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train(\"dev\")","metadata":{"id":"U0jJin_z384s","outputId":"efd3792d-3645-497f-b576-b708207e20dd"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Testing on dev data","metadata":{"id":"cxl8SDA5R8hz"}},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.metrics import roc_auc_score, mean_squared_error\n\ndef test(dataset_type: Literal[\"dev\", \"eval\"]):\n    assert dataset_type == \"dev\" or dataset_type == \"eval\"\n    with torch.no_grad():\n        model = CNNAutoEncoder()\n        model.load_state_dict(torch.load(f\"models/{dataset_type}.pt\"))\n        model.eval()\n        dataset = SpectrogramDataset(f\"{dataset_type}_data/{dataset_type}_data/slider/test\")\n        dataloader = torch.utils.data.DataLoader(dataset, batch_size=len(dataset))\n        \n        # we have only one batch in the dataloader\n        for test_spectrograms, labels, filenames in dataloader:\n            scores = []\n            preds = model(test_spectrograms.unsqueeze(1))\n            preds = preds.cpu().detach().numpy().squeeze()\n\n            # iterate through the batch\n            for i in range(len(preds)):\n                scores.append(mean_squared_error(test_spectrograms[i], preds[i]))\n            \n        if dataset_type == \"dev\":\n            auc = roc_auc_score(labels, scores)\n            print(\"AUC:\", auc)\n        else:\n            submission_df = pd.DataFrame()\n            submission_df[\"file_name\"] = list(map(lambda f: f+\".wav\", filenames))\n            submission_df[\"anomaly_score\"] = scores\n            submission_df.head()\n            submission_df.to_csv(\"submission.csv\", index=False)\n            print(\"Created submission.csv\")","metadata":{"id":"K903Ho4BHnqf"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test(\"dev\")","metadata":{"id":"KeNswcaVRmDq","outputId":"3b2959fb-bf79-4b2a-f45c-d9c71bbec461"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Visualize the reconstructed mel spectrogram:","metadata":{"id":"CYmLfF2-5FgB"}},{"cell_type":"code","source":"with torch.no_grad():\n    model = CNNAutoEncoder()\n    model.load_state_dict(torch.load(f\"models/dev.pt\"))\n    model.eval()\n    dataset = SpectrogramDataset(f\"dev_data/dev_data/slider/test\")\n    dataloader = torch.utils.data.DataLoader(dataset, batch_size=1)\n    \n    # we have only one batch in the dataloader\n    for test_spectrograms, labels, filenames in dataloader:\n        preds = model(test_spectrograms.unsqueeze(1))\n        preds = preds.cpu().detach().numpy().squeeze()\n\n        break\n\n# plot log_mel\nfig, ax = plt.subplots(1, 1, figsize=(6, 3))\nax.set_title(f\"Log Mel Spectrogram of {filename.split('/')[-1]}\")\nax.imshow(preds)\nax.axis('off')\nplt.show()            ","metadata":{"id":"-NgVENjAsUEP","outputId":"9f8d0dd2-0427-4227-ed02-7c6effad3464"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train and create submission on eval data","metadata":{"id":"4yAMkZcdz1Md"}},{"cell_type":"code","source":"train(\"eval\")","metadata":{"id":"yUovzgFwzuaX","outputId":"a1a438a8-edc3-4c92-c7bb-4035dffe84b2"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test(\"eval\")","metadata":{"id":"R6zZM-cd0WR0","outputId":"a7c776f7-18f7-4ab3-aa32-de9f8812369f"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! head -10 \"submission.csv\"","metadata":{"id":"3qNHlsrwJajX","outputId":"601df7bb-fc1b-4c2f-a017-2465ab5011f0"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Submit to kaggle with a message","metadata":{"id":"jg9nasZVNndR"}},{"cell_type":"code","source":"! kaggle competitions submit -c eurecom-aml-2022-challenge-2 -f submission.csv -m \"ConvAE v2\"","metadata":{"id":"KnFT0c9zNhen","outputId":"5612de08-5eed-4f9f-9c85-b4b11c3f2bd9"},"execution_count":null,"outputs":[]}]}