{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\nimport matplotlib.pyplot as plt\nfrom matplotlib import dates as md\nimport seaborn as sns\nimport plotly.graph_objs as go\nimport plotly\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\nimport cufflinks as cf\ncf.set_config_file(offline=True)\n\nfrom sklearn.metrics import r2_score\nfrom sklearn import metrics\n\nimport lightgbm as lgb\n\nfrom tqdm import tqdm\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from numpy.random import seed\nseed(42)\nimport tensorflow as tf\ntf.random.set_seed(42)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load dataset","metadata":{}},{"cell_type":"code","source":"path_GEPIII = '/kaggle/input/ashrae-energy-prediction'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_power_meter = pd.read_csv(os.path.join(path_GEPIII,'train.csv'))\ndf_meta = pd.read_csv(os.path.join(path_GEPIII,'building_metadata.csv'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_meta = df_meta.merge(df_power_meter[['building_id','meter']].drop_duplicates(), on='building_id')\ndf_meta['merged_id'] = df_meta['building_id'].astype('str') + '_' + df_meta['meter'].astype('str')\ndf_meta","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_power_meter = df_power_meter.pivot_table(index='timestamp', columns=['building_id','meter'], values='meter_reading')\ndf_power_meter.index = pd.to_datetime(df_power_meter.index)\ndf_power_meter.columns = df_power_meter.columns.get_level_values(0).astype('str')+'_'+df_power_meter.columns.get_level_values(1).astype('str')\ndf_power_meter","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_count = pd.DataFrame(df_power_meter.count()).reset_index().rename(columns={'index':'merged_id',0:'count'})\ndf_meta = df_meta.merge(df_count, on='merged_id')\ndf_meta","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train data: 54% of power meters (site0~9); Validation data10% of power meters (site10~12); test data: 36% of power meters ((site13~15))\ndf_power_meter = df_power_meter.loc['2016']\ntrain_data = df_power_meter.loc[:, df_meta.loc[(df_meta['site_id']<10)&(df_meta['count']>8784*0.9), 'merged_id']].copy()\nvalid_data = df_power_meter.loc[:, df_meta.loc[(df_meta['site_id']<13)&(df_meta['site_id']>=10)&(df_meta['count']>8784*0.9), 'merged_id']].copy()\ntest_data = df_power_meter.loc[:, df_meta.loc[df_meta['count']>8784*0.9, 'merged_id']].drop(train_data.columns, axis=1).copy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Normalize meter readings for each meter\ndef normalize(df):\n    mean = df.mean()\n    df -= mean\n    std = df.std()\n    df /= std\n    return df, mean, std\n\ntrain_value, train_mean, train_std = normalize(train_data)\nvalid_value, valid_mean, valid_std = normalize(valid_data)\ntest_value, test_mean, test_std = normalize(test_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Add noises\n+- 1,2 and 3 std, 0.1% respectively","metadata":{}},{"cell_type":"code","source":"# Add noises to train data\ntrain_value_noisy = train_value.copy()\ntrain_value_labels = train_value.copy()\n\nfor meter_name in tqdm(train_value.columns):\n    df_noisy_data = train_value_noisy[[meter_name]].copy()\n    df_noisy_data['noise'] = 0\n    \n    #Add noises (+-1,2,3 std)\n    std = df_noisy_data[meter_name].std()\n    for multiplier_std in [-3,-2,-1,1,2,3]:\n        random_hours = df_noisy_data[df_noisy_data['noise']==0].sample(frac=0.001, random_state=42).index\n        df_noisy_data.loc[random_hours, meter_name] = df_noisy_data.loc[random_hours, meter_name] + multiplier_std*std\n        df_noisy_data.loc[random_hours, 'noise'] = abs(multiplier_std)\n    \n    df_noisy_data[meter_name] = df_noisy_data[meter_name].fillna(method='ffill').fillna(method='bfill') \n    \n    train_value_noisy[meter_name] = df_noisy_data[meter_name].copy()    \n    train_value_labels[meter_name] = df_noisy_data['noise'].copy()    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add noises to valid data\nvalid_value_noisy = valid_value.copy()\nvalid_value_labels = valid_value.copy()\n\nfor meter_name in tqdm(valid_value.columns):\n    df_noisy_data = valid_value_noisy[[meter_name]].copy()\n    df_noisy_data['noise'] = 0\n    \n    #Add noises (+-1,2,3 std)\n    std = df_noisy_data[meter_name].std()\n    for multiplier_std in [-3,-2,-1,1,2,3]:\n        random_hours = df_noisy_data[df_noisy_data['noise']==0].sample(frac=0.001, random_state=42).index\n        df_noisy_data.loc[random_hours, meter_name] = df_noisy_data.loc[random_hours, meter_name] + multiplier_std*std\n        df_noisy_data.loc[random_hours, 'noise'] = abs(multiplier_std)\n    \n    df_noisy_data[meter_name] = df_noisy_data[meter_name].fillna(method='ffill').fillna(method='bfill') \n    \n    valid_value_noisy[meter_name] = df_noisy_data[meter_name].copy()     \n    valid_value_labels[meter_name] = df_noisy_data['noise'].copy()        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add noises to test data\ntest_value_noisy = test_value.copy()\ntest_value_labels = test_value.copy()\n\nfor meter_name in tqdm(test_value.columns):\n    df_noisy_data = test_value_noisy[[meter_name]].copy()\n    df_noisy_data['noise'] = 0\n    \n    #Add noises (+-1,2,3 std)\n    std = df_noisy_data[meter_name].std()\n    for multiplier_std in [-3,-2,-1,1,2,3]:\n        random_hours = df_noisy_data[df_noisy_data['noise']==0].sample(frac=0.001, random_state=42).index\n        df_noisy_data.loc[random_hours, meter_name] = df_noisy_data.loc[random_hours, meter_name] + multiplier_std*std\n        df_noisy_data.loc[random_hours, 'noise'] = abs(multiplier_std)\n    \n    df_noisy_data[meter_name] = df_noisy_data[meter_name].fillna(method='ffill').fillna(method='bfill') \n    \n    test_value_noisy[meter_name] = df_noisy_data[meter_name].copy()      \n    test_value_labels[meter_name] = df_noisy_data['noise'].copy()        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot of before and after adding noises\nfor meter_name in train_value.sample(n=10, axis=1, random_state=42).columns:\n    fig, axes = plt.subplots(1,2,figsize=(15,3))\n    \n    ymin = train_value_noisy[meter_name].min()*1.05\n    ymax = train_value_noisy[meter_name].max()*1.05\n    \n    train_value[meter_name].fillna(method='ffill').fillna(method='bfill').plot(title=meter_name+' (raw data)', ylim=(ymin, ymax),\n                                                                               ax=axes[0],color='blue')\n    train_value_noisy[meter_name].fillna(method='ffill').fillna(method='bfill').plot(title=meter_name+' (add noise)', ylim=(ymin, ymax),\n                                                                                     ax=axes[1],color='orange')\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build an autoencoder\nWe will build a convolutional reconstruction autoencoder model. The model will take input of shape (batch_size, sequence_length, num_features) and return output of the same shape.","metadata":{}},{"cell_type":"code","source":"# Prepare train data for autoencoder\ntrain_value = train_value.fillna(method='ffill').fillna(method='bfill') \ntrain_value = train_value.dropna(axis=1, how='all')\n\nx_train = np.reshape(train_value.T.values, train_value.T.values.shape+(1,))\nx_train.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare noised train data for autoencoder\ntrain_value_noisy = train_value_noisy[train_value.columns]\ntrain_value_noisy = train_value_noisy.fillna(method='ffill').fillna(method='bfill') \n\nx_train_noisy = np.reshape(train_value_noisy.T.values, train_value_noisy.T.values.shape+(1,))\nx_train_noisy.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare test data for autoencoder\ntest_value = test_value.fillna(method='ffill').fillna(method='bfill') \ntest_value = test_value.dropna(axis=1, how='all')\n\nx_test = np.reshape(test_value.T.values, test_value.T.values.shape+(1,))\nx_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare noised test data for autoencoder\ntest_value_noisy = test_value_noisy[test_value.columns]\ntest_value_noisy = test_value_noisy.fillna(method='ffill').fillna(method='bfill') \n\nx_test_noisy = np.reshape(test_value_noisy.T.values, test_value_noisy.T.values.shape+(1,))\nx_test_noisy.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare valid data for autoencoder\nvalid_value = valid_value.fillna(method='ffill').fillna(method='bfill') \nvalid_value = valid_value.dropna(axis=1, how='all')\n\nx_valid = np.reshape(valid_value.T.values, valid_value.T.values.shape+(1,))\nx_valid.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare noised valid data for autoencoder\nvalid_value_noisy = valid_value_noisy[valid_value.columns]\nvalid_value_noisy = valid_value_noisy.fillna(method='ffill').fillna(method='bfill') \n\nx_valid_noisy = np.reshape(valid_value_noisy.T.values, valid_value_noisy.T.values.shape+(1,))\nx_valid_noisy.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Build 1D CNN autoencoder\nmodel = keras.Sequential(\n    [\n        layers.Input(shape=(x_train.shape[1], x_train.shape[2])),\n        layers.Conv1D(\n            filters=32, kernel_size=7, padding=\"same\", strides=2, activation=\"relu\"\n        ),\n        layers.Dropout(rate=0.2),\n        layers.Conv1D(\n            filters=16, kernel_size=7, padding=\"same\", strides=2, activation=\"relu\"\n        ),\n        layers.Conv1DTranspose(\n            filters=16, kernel_size=7, padding=\"same\", strides=2, activation=\"relu\"\n        ),\n        layers.Dropout(rate=0.2),\n        layers.Conv1DTranspose(\n            filters=32, kernel_size=7, padding=\"same\", strides=2, activation=\"relu\"\n        ),\n        layers.Conv1DTranspose(filters=1, kernel_size=7, padding=\"same\"),\n    ]\n)\nmodel.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001), loss=\"mae\")\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.utils.vis_utils import plot_model\nplot_model(model, to_file='model_plot.png', show_shapes=True, show_layer_names=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train the model\n- Input: noised train data\n- Output: train data","metadata":{}},{"cell_type":"code","source":"history = model.fit(\n    x_train_noisy,\n    x_train,\n    epochs=50,\n    batch_size=128,\n    #validation_split=0.20,\n    validation_data=(x_valid_noisy, x_valid),\n    #shuffle=False,\n    callbacks=[\n        keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=5, mode=\"min\")\n    ],\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history[\"loss\"], label=\"Training Loss\")\nplt.plot(history.history[\"val_loss\"], label=\"Validation Loss\")\nplt.ylim(0,1)\n\nplt.legend()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get train MAE loss\nx_train_pred = model.predict(x_train)\ntrain_mae_loss = np.mean(np.abs(x_train_pred - x_train), axis=1)\n\nplt.hist(train_mae_loss, bins=50)\nplt.xlabel(\"Train MAE loss\")\nplt.ylabel(\"No of samples\")\nplt.show()\n\navg = np.mean(train_mae_loss)\nprint(\"Reconstruction error average: \", avg)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot of reconstructed result (train data)\nfor idx in np.arange(0,1000,100):\n    df_plot = pd.concat([pd.Series(x_train[idx].flatten()).rename('x_train'), pd.Series(x_train_noisy[idx].flatten()).rename('x_train_noisy'), pd.Series(x_train_pred[idx].flatten()).rename('x_train_pred')], axis=1)\n    df_plot['labels'] = train_value_labels.iloc[:, idx].values\n    df_plot['Squared error'] = (df_plot['x_train_pred'] - df_plot['x_train_noisy'])**2\n    df_plot['Squared error'] = df_plot['Squared error']/df_plot['Squared error'].std()\n    df_plot.iplot()\n    #pd.concat([pd.Series(x_train_noisy[idx].flatten()).rename('x_train_noisy'), pd.Series(x_train_pred[idx].flatten()).rename('x_train_pred')], axis=1).plot(figsize=(20,3), alpha=0.7)\n    plt.show()\n    #sns.displot(df_plot[df_plot['labels']>0], x=\"Squared error\", hue=\"labels\", kind=\"kde\")\n    #plt.xlim(-5,20)\n    #plt.show()    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Apply trained autoencoder on test data","metadata":{}},{"cell_type":"code","source":"# Get test MAE loss\nx_test_pred = model.predict(x_test_noisy)\ntest_mae_loss = np.mean(np.abs(x_test_pred - x_test), axis=1)\n\ntest_pred = test_value.copy()\ntest_pred.loc[:,:] = x_test_pred[:,:,0].T\n\nplt.hist(test_mae_loss, bins=50)\nplt.xlabel(\"test MAE loss\")\nplt.ylabel(\"No of samples\")\nplt.show()\n\navg = np.mean(test_mae_loss)\nprint(\"Reconstruction error average: \", avg)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_plot","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot of reconstructed result (test data)\nfor idx in np.arange(0,300,20):\n    df_plot = pd.concat([pd.Series(x_test[idx].flatten()).rename('x_test'), pd.Series(x_test_noisy[idx].flatten()).rename('x_test_noisy'), pd.Series(x_test_pred[idx].flatten()).rename('x_test_pred')], axis=1)\n    df_plot['labels'] = test_value_labels.iloc[:, idx].values\n    df_plot['Squared error'] = (df_plot['x_test_pred'] - df_plot['x_test_noisy'])**2\n    df_plot['Squared error'] = df_plot['Squared error']/df_plot['Squared error'].std()\n    df_plot.iplot()\n    #pd.concat([pd.Series(x_test_noisy[idx].flatten()).rename('x_test_noisy'), pd.Series(x_test_pred[idx].flatten()).rename('x_test_pred')], axis=1).plot(figsize=(20,3), alpha=0.7)\n    plt.show()    \n    sns.displot(df_plot[df_plot['labels']>0], x=\"Squared error\", hue=\"labels\", kind=\"kde\")\n    plt.xlim(-5,20)\n    plt.show()    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_valid_pred = model.predict(x_valid_noisy)\nvalid_mae_loss = np.mean(np.abs(x_valid_pred - x_valid), axis=1)\n\nvalid_pred = valid_value.copy()\nvalid_pred.loc[:,:] = x_valid_pred[:,:,0].T","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"error_train = []\n\n# Plot of reconstructed result (train data)\nfor idx in tqdm(np.arange(0,len(x_train))):\n    df_plot = pd.concat([pd.Series(x_train[idx].flatten()).rename('x_train'), pd.Series(x_train_noisy[idx].flatten()).rename('x_train_noisy'), pd.Series(x_train_pred[idx].flatten()).rename('x_train_pred')], axis=1)\n    df_plot['labels'] = train_value_labels.iloc[:, idx].values\n    df_plot['Squared error'] = (df_plot['x_train_pred'] - df_plot['x_train_noisy'])**2\n    df_plot['Squared error'] = df_plot['Squared error']/df_plot['Squared error'].std()\n    df_plot['merged_id'] = train_value_labels.columns[idx]\n    error_train.append(df_plot)\n    \nerror_train = pd.concat(error_train,axis=0,ignore_index=True)\nerror_train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"error_test = []\n\n# Plot of reconstructed result (test data)\nfor idx in tqdm(np.arange(0,len(x_test))):\n    df_plot = pd.concat([pd.Series(x_test[idx].flatten()).rename('x_test'), pd.Series(x_test_noisy[idx].flatten()).rename('x_test_noisy'), pd.Series(x_test_pred[idx].flatten()).rename('x_test_pred')], axis=1)\n    df_plot['labels'] = test_value_labels.iloc[:, idx].values\n    df_plot['Squared error'] = (df_plot['x_test_pred'] - df_plot['x_test_noisy'])**2\n    df_plot['Squared error'] = df_plot['Squared error']/df_plot['Squared error'].std()\n    df_plot['merged_id'] = test_value_labels.columns[idx]\n    error_test.append(df_plot)\n    \nerror_test = pd.concat(error_test,axis=0,ignore_index=True)\nerror_test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"error_valid = []\n\n# Plot of reconstructed result (valid data)\nfor idx in tqdm(np.arange(0,len(x_valid))):\n    df_plot = pd.concat([pd.Series(x_valid[idx].flatten()).rename('x_valid'), pd.Series(x_valid_noisy[idx].flatten()).rename('x_valid_noisy'), pd.Series(x_valid_pred[idx].flatten()).rename('x_valid_pred')], axis=1)\n    df_plot['labels'] = valid_value_labels.iloc[:, idx].values\n    df_plot['Squared error'] = (df_plot['x_valid_pred'] - df_plot['x_valid_noisy'])**2\n    df_plot['Squared error'] = df_plot['Squared error']/df_plot['Squared error'].std()\n    df_plot['merged_id'] = valid_value_labels.columns[idx]\n    error_valid.append(df_plot)\n    \nerror_valid = pd.concat(error_valid,axis=0,ignore_index=True)\nerror_valid","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(error_train[error_train['labels']>0], x=\"Squared error\", hue=\"labels\", kind=\"kde\")\nplt.xlim(-5,20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(error_valid[error_valid['labels']>0], x=\"Squared error\", hue=\"labels\", kind=\"kde\")\nplt.xlim(-5,20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(error_test[error_test['labels']>0].sample(10000), x=\"Squared error\", hue=\"labels\", kind=\"kde\")\nplt.xlim(-5,20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"error_valid['noise'] = (abs(error_valid['labels'])>0).astype('int')\nfpr, tpr, thresholds = metrics.roc_curve(error_valid['noise'], error_valid['Squared error'], pos_label=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nfrom sklearn.metrics import precision_recall_curve\nfrom sklearn.metrics import PrecisionRecallDisplay\nfrom sklearn.metrics import RocCurveDisplay\n\nprecision, recall, thresholds = precision_recall_curve(error_valid['noise'], error_valid['Squared error'], pos_label=1)\npr_display = PrecisionRecallDisplay(precision=precision, recall=recall).plot()\nroc_display = RocCurveDisplay(fpr=fpr, tpr=tpr).plot()\n\noptimal_idx = np.argmax(2*precision*recall/(precision+recall))\noptimal_threshold = thresholds[optimal_idx]\nprint(\"AUC is:\", metrics.auc(fpr, tpr))\nprint(\"Threshold value is:\", optimal_threshold)\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_roc_curve(fper, tper):\n    plt.plot(fper, tper, color='red', label='ROC')\n    plt.plot([0, 1], [0, 1], color='green', linestyle='--')\n    plt.xlabel('False Positive Rate')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver Operating Characteristic Curve')\n    plt.legend()\n    plt.show()\n    \noptimal_idx = np.argmax(tpr - fpr)\noptimal_threshold = thresholds[optimal_idx]\nprint(\"AUC is:\", metrics.auc(fpr, tpr))\nprint(\"Threshold value is:\", optimal_threshold)\nplot_roc_curve(fpr, tpr)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"error_test['noise'] = (abs(error_test['labels'])>0).astype('int')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"error_valid.groupby('labels')['Squared error'].agg(['mean','std'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"error_test.sample(frac=0.1, random_state=42).groupby('labels')['Squared error'].agg(['mean','std'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\nfrom sklearn.metrics import f1_score\n\nfor optimal_threshold in np.arange(3.5,5.5,0.1):\n    error_valid['detection'] = (error_valid['Squared error']>optimal_threshold).astype('int')\n    f1 = f1_score(error_valid['noise'], error_valid['detection'])\n\n    cm = confusion_matrix(error_valid['noise'], error_valid['detection'], labels=[0,1])\n    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=[0,1])\n    disp.plot()\n    plt.title('threshold: '+str(round(optimal_threshold,1))+' // f1 score: '+ str(round(f1,5)))\n    plt.show()\n    display(error_valid.pivot_table(index='detection',columns=['labels'],values='merged_id',aggfunc='count'))    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"record_merged = []\n\nfor optimal_threshold in np.arange(1.0,8.0,0.1):\n    error_valid['detection'] = (error_valid['Squared error']>optimal_threshold).astype('int')\n    f1 = f1_score(error_valid['noise'], error_valid['detection'])\n\n    cm = confusion_matrix(error_valid['noise'], error_valid['detection'], labels=[0,1])  \n    \n    record = pd.DataFrame(data=cm.reshape(1,-4),columns=['TN', 'FN', 'FP', 'TP'])\n    record['optimal_threshold'] = optimal_threshold\n    record['f1'] = f1    \n    \n    record_merged.append(record)    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"record_merged=pd.concat(record_merged,axis=0,ignore_index=True)\nrecord_merged['Precision'] = record_merged['TP']/(record_merged['TP']+record_merged['FP'])\nrecord_merged['Recall'] = record_merged['TP']/(record_merged['TP']+record_merged['FP'])\nrecord_merged['FP + FN'] = record_merged['FP'] + record_merged['FN']\n\nrecord_merged.set_index('optimal_threshold')[['FN','FP', 'FP + FN']].plot(ylabel='counts')\nrecord_merged.set_index('optimal_threshold')[['f1']].plot(ylabel='f1 score')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimal_threshold = 5.0\n\nerror_test['detection'] = (error_test['Squared error']>optimal_threshold).astype('int')\nf1 = f1_score(error_test['noise'], error_test['detection'])\n\ncm = confusion_matrix(error_test['noise'], error_test['detection'], labels=[0,1])\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=[0,1])\ndisp.plot()\nplt.title('threshold: '+str(round(optimal_threshold,1))+' // f1 score: '+ str(round(f1,4)))\nplt.show()\ndisplay(error_test.pivot_table(index='detection',columns=['labels'],values='merged_id',aggfunc='count'))   ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for merged_id in error_test['merged_id'].unique()[::20]:\n    df_plot = error_test.loc[error_test['merged_id']==merged_id, ['x_test_noisy', 'x_test_pred']].copy()\n    df_plot['detected anomalies'] = error_test.loc[error_test['merged_id']==merged_id, :].reset_index().pivot_table(index='index',columns='detection',values='x_test_noisy')[1].values\n    df_plot = df_plot.iloc[:4000]\n    plt.figure(figsize=(8,6))\n    plt.plot(df_plot.index, df_plot['x_test_noisy'], '-b', label='x_test_noisy')\n    plt.plot(df_plot.index, df_plot['x_test_pred'], '--g', label='x_test_pred')\n    plt.plot(df_plot.index, df_plot['detected anomalies'], 'r*', label='detected anomalies')    \n    plt.title(merged_id)\n    plt.legend(loc='upper right')    \n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for merged_id in error_test['merged_id'].unique()[::20]:\n    df_plot = error_test.loc[error_test['merged_id']==merged_id, ['x_test_noisy', 'x_test_pred']].copy()\n    df_plot['detected anomalies'] = error_test.loc[error_test['merged_id']==merged_id, :].reset_index().pivot_table(index='index',columns='detection',values='x_test_noisy')[1].values\n    df_plot = df_plot.iloc[5000:5500]\n    plt.figure(figsize=(8,6))\n    plt.plot(df_plot.index, df_plot['x_test_noisy'], '-b', label='x_test_noisy')\n    plt.plot(df_plot.index, df_plot['x_test_pred'], '--g', label='x_test_pred')\n    plt.plot(df_plot.index, df_plot['detected anomalies'], 'r*', label='detected anomalies')    \n    plt.title(merged_id)\n    plt.legend(loc='upper right')    \n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}