{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Anomaly detection by autoencoder with Keras\n\nI'm a beginner in deep learning, so I'm sharing what I've learned.  I used [this blog](https://blog.keras.io/building-autoencoders-in-keras.html) as a reference.\n\nThe train data should be trained by autoencoder with \"1\" only, and the test data should be \"0\" and \"1\" only. For the restored image, check that \"0\" is not restored. Check the difference from the original image, and \"0\" can be detected as abnormal data.\n## Setup","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nfrom keras.layers import Input, Dense, Conv2D, MaxPooling2D, UpSampling2D\nfrom keras.models import Model\nfrom keras import regularizers","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-16T04:02:11.345925Z","iopub.execute_input":"2021-08-16T04:02:11.346496Z","iopub.status.idle":"2021-08-16T04:02:11.354496Z","shell.execute_reply.started":"2021-08-16T04:02:11.346462Z","shell.execute_reply":"2021-08-16T04:02:11.353730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to display images\ndef disp_img(img):\n    n = 10  \n    plt.figure(figsize=(20, 4))\n    for i in range(n):\n        ax = plt.subplot(2, n, i + 1)\n        plt.imshow(img[i].reshape(28, 28))\n        plt.gray()\n        ax.get_xaxis().set_visible(False)\n        ax.get_yaxis().set_visible(False)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:02:12.701775Z","iopub.execute_input":"2021-08-16T04:02:12.702266Z","iopub.status.idle":"2021-08-16T04:02:12.707641Z","shell.execute_reply.started":"2021-08-16T04:02:12.702235Z","shell.execute_reply":"2021-08-16T04:02:12.706909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(42)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:02:13.901518Z","iopub.execute_input":"2021-08-16T04:02:13.902023Z","iopub.status.idle":"2021-08-16T04:02:13.906545Z","shell.execute_reply.started":"2021-08-16T04:02:13.901991Z","shell.execute_reply":"2021-08-16T04:02:13.905646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Preparation\nUse MNIST data from Digit Recognizer.","metadata":{}},{"cell_type":"code","source":"temp = pd.read_csv('../input/digit-recognizer/train.csv')\ndf01 = temp[temp['label'] < 2]\ndel temp\n\ndf_train = df01[df01['label'] == 1].drop(['label'],axis = 1)  #Train data is \"1\" only\ntrain = df_train.values\ntrain = train.astype('float32') / 255.\n\ndf_test = df01.drop(['label'],axis = 1).head(10)  #Test data is \"0\" and \"1\"\ntest = df_test.values\ntest = test.astype('float32') / 255.\ndel df01","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:02:15.675429Z","iopub.execute_input":"2021-08-16T04:02:15.675778Z","iopub.status.idle":"2021-08-16T04:02:18.651371Z","shell.execute_reply.started":"2021-08-16T04:02:15.675750Z","shell.execute_reply":"2021-08-16T04:02:18.650397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train data is **\"1\"** only","metadata":{}},{"cell_type":"code","source":"print(train.shape)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:02:18.652919Z","iopub.execute_input":"2021-08-16T04:02:18.653260Z","iopub.status.idle":"2021-08-16T04:02:18.658995Z","shell.execute_reply.started":"2021-08-16T04:02:18.653230Z","shell.execute_reply":"2021-08-16T04:02:18.657836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"disp_img(train)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:02:18.771826Z","iopub.execute_input":"2021-08-16T04:02:18.772215Z","iopub.status.idle":"2021-08-16T04:02:19.343448Z","shell.execute_reply.started":"2021-08-16T04:02:18.772184Z","shell.execute_reply":"2021-08-16T04:02:19.342336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Test data is **\"0\" and \"1\"**","metadata":{}},{"cell_type":"code","source":"print(test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:02:20.581323Z","iopub.execute_input":"2021-08-16T04:02:20.581672Z","iopub.status.idle":"2021-08-16T04:02:20.586731Z","shell.execute_reply.started":"2021-08-16T04:02:20.581644Z","shell.execute_reply":"2021-08-16T04:02:20.585671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"disp_img(test)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:02:21.364964Z","iopub.execute_input":"2021-08-16T04:02:21.365475Z","iopub.status.idle":"2021-08-16T04:02:22.121017Z","shell.execute_reply.started":"2021-08-16T04:02:21.365442Z","shell.execute_reply":"2021-08-16T04:02:22.120316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fully-connected neural layer\nIt was best to use binary cross-entropy as the loss function and adam as the optimization method.","metadata":{}},{"cell_type":"code","source":"def create_fullautoencoder(train, opt, loss):\n    input_img = Input(shape=(784, ))\n    encoded = Dense(32, activation='relu')(input_img)\n    decoded = Dense(784, activation='sigmoid')(encoded)\n    autoencoder = Model(input_img, decoded)\n    autoencoder.compile(optimizer=opt, loss=loss)\n    return(autoencoder)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:02:25.250270Z","iopub.execute_input":"2021-08-16T04:02:25.250808Z","iopub.status.idle":"2021-08-16T04:02:25.256310Z","shell.execute_reply.started":"2021-08-16T04:02:25.250776Z","shell.execute_reply":"2021-08-16T04:02:25.255320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Differences in restored images due to optimization methods\nThe loss function is fixed to binary cross-entropy, and the differences between the restored images from the optimization methods are compared.\n#### SGD\nStochastic Gradient Descent. Easy, but zigzagging is a violent movement and in some cases inefficient.  \nSGD can't even reproduce \"1\"...","metadata":{}},{"cell_type":"code","source":"autoencoder1 = create_fullautoencoder(train, 'sgd', 'binary_crossentropy')\nautoencoder1.fit(train, train, epochs=50, batch_size=256, shuffle=True)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-08-16T04:02:28.374431Z","iopub.execute_input":"2021-08-16T04:02:28.374789Z","iopub.status.idle":"2021-08-16T04:02:35.616461Z","shell.execute_reply.started":"2021-08-16T04:02:28.374762Z","shell.execute_reply":"2021-08-16T04:02:35.615624Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"autoencoder1.summary()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:03:02.031698Z","iopub.execute_input":"2021-08-16T04:03:02.032059Z","iopub.status.idle":"2021-08-16T04:03:02.039673Z","shell.execute_reply.started":"2021-08-16T04:03:02.032027Z","shell.execute_reply":"2021-08-16T04:03:02.038422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoded1 = autoencoder1.predict(test)\ndisp_img(test)\ndisp_img(decoded1)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:03:16.624982Z","iopub.execute_input":"2021-08-16T04:03:16.625392Z","iopub.status.idle":"2021-08-16T04:03:17.875110Z","shell.execute_reply.started":"2021-08-16T04:03:16.625362Z","shell.execute_reply":"2021-08-16T04:03:17.874139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Adagrad\nA method to automatically determine the learning rate.  Adagrad didn't work either...","metadata":{}},{"cell_type":"code","source":"autoencoder2 = create_fullautoencoder(train, 'adagrad', 'binary_crossentropy')\nautoencoder2.fit(train, train, epochs=50, batch_size=256, shuffle=True)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-08-16T04:03:31.662858Z","iopub.execute_input":"2021-08-16T04:03:31.663212Z","iopub.status.idle":"2021-08-16T04:03:38.579891Z","shell.execute_reply.started":"2021-08-16T04:03:31.663183Z","shell.execute_reply":"2021-08-16T04:03:38.578756Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoded2 = autoencoder2.predict(test)\ndisp_img(test)\ndisp_img(decoded2)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T02:05:00.894044Z","iopub.execute_input":"2021-08-16T02:05:00.894384Z","iopub.status.idle":"2021-08-16T02:05:01.931495Z","shell.execute_reply.started":"2021-08-16T02:05:00.894357Z","shell.execute_reply":"2021-08-16T02:05:01.930005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Adadelta\nPrevent rapid and monotonous decline in learning rate with AdaGrad's developmental system.  ","metadata":{}},{"cell_type":"code","source":"autoencoder3 = create_fullautoencoder(train, 'adadelta', 'binary_crossentropy')\nautoencoder3.fit(train, train, epochs=50, batch_size=256, shuffle=True)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-08-16T02:05:01.934121Z","iopub.execute_input":"2021-08-16T02:05:01.934429Z","iopub.status.idle":"2021-08-16T02:05:08.759034Z","shell.execute_reply.started":"2021-08-16T02:05:01.934405Z","shell.execute_reply":"2021-08-16T02:05:08.758201Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoded3 = autoencoder3.predict(test)\ndisp_img(test)\ndisp_img(decoded3)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T02:05:08.761883Z","iopub.execute_input":"2021-08-16T02:05:08.762135Z","iopub.status.idle":"2021-08-16T02:05:09.865123Z","shell.execute_reply.started":"2021-08-16T02:05:08.762103Z","shell.execute_reply":"2021-08-16T02:05:09.863920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### RMSprop\nA method that solves the shortcomings of Adagrad by accumulating exponential moving averages.  \n**Good job! As hoped, we can determine that the \"0\" is an anomaly because we can reproduce the \"1\" and not the \"0\".**","metadata":{}},{"cell_type":"code","source":"autoencoder4 = create_fullautoencoder(train, 'rmsprop', 'binary_crossentropy')\nautoencoder4.fit(train, train, epochs=50, batch_size=256, shuffle=True)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-08-16T02:05:09.867006Z","iopub.execute_input":"2021-08-16T02:05:09.867480Z","iopub.status.idle":"2021-08-16T02:05:16.942778Z","shell.execute_reply.started":"2021-08-16T02:05:09.867443Z","shell.execute_reply":"2021-08-16T02:05:16.941748Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoded4 = autoencoder4.predict(test)\ndisp_img(test)\ndisp_img(decoded4)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T02:05:16.943969Z","iopub.execute_input":"2021-08-16T02:05:16.944279Z","iopub.status.idle":"2021-08-16T02:05:18.004152Z","shell.execute_reply.started":"2021-08-16T02:05:16.944247Z","shell.execute_reply":"2021-08-16T02:05:18.003127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Adam\nIt's a combination of RMSprop and Momentum, but with some differences.  \n**Good job! As hoped, we can determine that the \"0\" is an anomaly because we can reproduce the \"1\" and not the \"0\".**","metadata":{}},{"cell_type":"code","source":"autoencoder5 = create_fullautoencoder(train, 'adam', 'binary_crossentropy')\nautoencoder5.fit(train, train, epochs=50, batch_size=256, shuffle=True)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-08-16T02:08:49.153246Z","iopub.execute_input":"2021-08-16T02:08:49.153600Z","iopub.status.idle":"2021-08-16T02:08:55.779543Z","shell.execute_reply.started":"2021-08-16T02:08:49.153571Z","shell.execute_reply":"2021-08-16T02:08:55.778558Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoded5 = autoencoder5.predict(test)\ndisp_img(test)\ndisp_img(decoded5)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T02:08:55.781255Z","iopub.execute_input":"2021-08-16T02:08:55.781545Z","iopub.status.idle":"2021-08-16T02:08:56.929983Z","shell.execute_reply.started":"2021-08-16T02:08:55.781521Z","shell.execute_reply":"2021-08-16T02:08:56.928631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Nadam\nAdam's take on Nesterov's Accelerated Gradient Method. This is ok too.","metadata":{}},{"cell_type":"code","source":"autoencoder6 = create_fullautoencoder(train, 'nadam', 'binary_crossentropy')\nautoencoder6.fit(train, train, epochs=50, batch_size=256, shuffle=True)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-08-16T02:05:26.072272Z","iopub.execute_input":"2021-08-16T02:05:26.072555Z","iopub.status.idle":"2021-08-16T02:05:33.400733Z","shell.execute_reply.started":"2021-08-16T02:05:26.072527Z","shell.execute_reply":"2021-08-16T02:05:33.399818Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoded6 = autoencoder6.predict(test)\ndisp_img(test)\ndisp_img(decoded6)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T02:05:33.401671Z","iopub.execute_input":"2021-08-16T02:05:33.401879Z","iopub.status.idle":"2021-08-16T02:05:34.494175Z","shell.execute_reply.started":"2021-08-16T02:05:33.401856Z","shell.execute_reply":"2021-08-16T02:05:34.493059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Difference of restored image by loss function\nThe optimization method is fixed to adam, and the differences in the restored images due to the loss function are compared.\n#### MSE\nMean square error. Successful anomaly detection.","metadata":{}},{"cell_type":"code","source":"autoencoder7 = create_fullautoencoder(train, 'adam', 'mean_squared_error')\nautoencoder7.fit(train, train, epochs=50, batch_size=256, shuffle=True)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-08-16T02:05:34.495184Z","iopub.execute_input":"2021-08-16T02:05:34.495402Z","iopub.status.idle":"2021-08-16T02:05:38.634629Z","shell.execute_reply.started":"2021-08-16T02:05:34.495378Z","shell.execute_reply":"2021-08-16T02:05:38.633621Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoded7 = autoencoder7.predict(test)\ndisp_img(test)\ndisp_img(decoded7)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T02:05:38.635634Z","iopub.execute_input":"2021-08-16T02:05:38.635888Z","iopub.status.idle":"2021-08-16T02:05:39.470390Z","shell.execute_reply.started":"2021-08-16T02:05:38.635862Z","shell.execute_reply":"2021-08-16T02:05:39.469072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Binary crossentropy\nAgain, we will run it again to compare it with the other loss functions.  \nSuccessful anomaly detection.","metadata":{}},{"cell_type":"code","source":"autoencoder8 = create_fullautoencoder(train, 'adam', 'binary_crossentropy')\nautoencoder8.fit(train, train, epochs=50, batch_size=256, shuffle=True)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-08-16T02:21:39.349336Z","iopub.execute_input":"2021-08-16T02:21:39.349664Z","iopub.status.idle":"2021-08-16T02:21:45.998466Z","shell.execute_reply.started":"2021-08-16T02:21:39.349635Z","shell.execute_reply":"2021-08-16T02:21:45.997823Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoded8 = autoencoder8.predict(test)\ndisp_img(test)\ndisp_img(decoded8)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T02:21:45.999295Z","iopub.execute_input":"2021-08-16T02:21:45.999481Z","iopub.status.idle":"2021-08-16T02:21:46.823066Z","shell.execute_reply.started":"2021-08-16T02:21:45.999461Z","shell.execute_reply":"2021-08-16T02:21:46.822086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### KL divergence\nIt didn't work at all.","metadata":{}},{"cell_type":"code","source":"autoencoder9 = create_fullautoencoder(train, 'adam', 'kullback_leibler_divergence')\nautoencoder9.fit(train, train, epochs=50, batch_size=256, shuffle=True)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-08-16T02:22:28.805420Z","iopub.execute_input":"2021-08-16T02:22:28.805762Z","iopub.status.idle":"2021-08-16T02:22:35.218893Z","shell.execute_reply.started":"2021-08-16T02:22:28.805733Z","shell.execute_reply":"2021-08-16T02:22:35.218015Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoded9 = autoencoder9.predict(test)\ndisp_img(test)\ndisp_img(decoded9)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T02:22:51.645564Z","iopub.execute_input":"2021-08-16T02:22:51.645845Z","iopub.status.idle":"2021-08-16T02:22:52.460166Z","shell.execute_reply.started":"2021-08-16T02:22:51.645822Z","shell.execute_reply":"2021-08-16T02:22:52.459042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sparse autoencoder\nAdds sparse regularization to a simple autoencoder.  \n\nFor fully connected neural layers, the best results were obtained using binary cross entropy as the loss function and adam as the optimization method, so we will use this combination in the future. \n\nSuccessful anomaly detection.","metadata":{}},{"cell_type":"code","source":"def create_sparseautoencoder(train, opt, loss):\n    input_img = Input(shape=(784, ))\n    encoded = Dense(32, activation='relu', \n                    activity_regularizer=regularizers.l1(10e-5))(input_img)\n    decoded = Dense(784, activation='sigmoid')(encoded)\n    autoencoder = Model(input_img, decoded)\n    autoencoder.compile(optimizer=opt, loss=loss)\n    return(autoencoder)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:04:06.331910Z","iopub.execute_input":"2021-08-16T04:04:06.332391Z","iopub.status.idle":"2021-08-16T04:04:06.338009Z","shell.execute_reply.started":"2021-08-16T04:04:06.332353Z","shell.execute_reply":"2021-08-16T04:04:06.337208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"autoencoder10 = create_sparseautoencoder(train, 'adam', 'binary_crossentropy')\nautoencoder10.fit(train, train, epochs=50, batch_size=256, shuffle=True)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-08-16T04:04:10.240726Z","iopub.execute_input":"2021-08-16T04:04:10.241254Z","iopub.status.idle":"2021-08-16T04:04:17.511498Z","shell.execute_reply.started":"2021-08-16T04:04:10.241221Z","shell.execute_reply":"2021-08-16T04:04:17.510415Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"autoencoder10.summary()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:04:22.099261Z","iopub.execute_input":"2021-08-16T04:04:22.099608Z","iopub.status.idle":"2021-08-16T04:04:22.106345Z","shell.execute_reply.started":"2021-08-16T04:04:22.099579Z","shell.execute_reply":"2021-08-16T04:04:22.105401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoded10 = autoencoder10.predict(test)\ndisp_img(test)\ndisp_img(decoded10)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:04:29.048727Z","iopub.execute_input":"2021-08-16T04:04:29.049217Z","iopub.status.idle":"2021-08-16T04:04:30.422154Z","shell.execute_reply.started":"2021-08-16T04:04:29.049186Z","shell.execute_reply":"2021-08-16T04:04:30.421247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Deep autoencoder\nSuccessful anomaly detection.","metadata":{}},{"cell_type":"code","source":"def create_deepautoencoder(train, opt, loss):\n    input_img = Input(shape=(784,))\n    encoded = Dense(128, activation='relu')(input_img)\n    encoded = Dense(64, activation='relu')(encoded)\n    encoded = Dense(32, activation='relu')(encoded)\n    decoded = Dense(64, activation='relu')(encoded)\n    decoded = Dense(128, activation='relu')(decoded)\n    decoded = Dense(784, activation='sigmoid')(decoded)\n    autoencoder = Model(input_img, decoded)\n    autoencoder.compile(optimizer=opt, loss=loss)\n    return(autoencoder)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:06:18.120076Z","iopub.execute_input":"2021-08-16T04:06:18.120471Z","iopub.status.idle":"2021-08-16T04:06:18.127070Z","shell.execute_reply.started":"2021-08-16T04:06:18.120440Z","shell.execute_reply":"2021-08-16T04:06:18.126348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"autoencoder11 = create_deepautoencoder(train, 'adam', 'binary_crossentropy')\nautoencoder11.fit(train, train, epochs=50, batch_size=256, shuffle=True)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-08-16T04:06:25.183383Z","iopub.execute_input":"2021-08-16T04:06:25.183940Z","iopub.status.idle":"2021-08-16T04:06:35.447173Z","shell.execute_reply.started":"2021-08-16T04:06:25.183872Z","shell.execute_reply":"2021-08-16T04:06:35.446248Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"autoencoder11.summary()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:06:37.698886Z","iopub.execute_input":"2021-08-16T04:06:37.699241Z","iopub.status.idle":"2021-08-16T04:06:37.708030Z","shell.execute_reply.started":"2021-08-16T04:06:37.699212Z","shell.execute_reply":"2021-08-16T04:06:37.706329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoded11 = autoencoder11.predict(test)\ndisp_img(test)\ndisp_img(decoded11)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:06:43.358302Z","iopub.execute_input":"2021-08-16T04:06:43.358698Z","iopub.status.idle":"2021-08-16T04:06:44.588273Z","shell.execute_reply.started":"2021-08-16T04:06:43.358666Z","shell.execute_reply":"2021-08-16T04:06:44.587108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Convolutional autoencoder\nThis is an auto-encoder that uses CNN.  \n\nCNN even goes so far as to restore the \"0\". Therefore, a simple black and white image like MNIST may be difficult to detect anomalies, but may be suitable for anomaly detection of fine details like cracks in concrete.","metadata":{}},{"cell_type":"code","source":"def create_deepautoencoder(train, opt, loss):\n    input_img = Input(shape=(28, 28, 1))\n    x = Conv2D(16, (3, 3), activation='relu', padding='same')(input_img)\n    x = MaxPooling2D((2, 2), padding='same')(x)\n    x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)\n    x = MaxPooling2D((2, 2), padding='same')(x)\n    x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)\n    encoded = MaxPooling2D((2, 2), padding='same')(x)\n    \n    x = Conv2D(8, (3, 3), activation='relu', padding='same')(encoded)\n    x = UpSampling2D((2, 2))(x)\n    x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)\n    x = UpSampling2D((2, 2))(x)\n    x = Conv2D(16, (3, 3), activation='relu')(x)\n    x = UpSampling2D((2, 2))(x)\n    decoded = Conv2D(1, (3, 3), activation='sigmoid', padding='same')(x)\n    \n    autoencoder = Model(input_img, decoded)\n    autoencoder.compile(optimizer='adam', loss='binary_crossentropy')\n    return(autoencoder)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-08-16T04:09:52.952520Z","iopub.execute_input":"2021-08-16T04:09:52.952873Z","iopub.status.idle":"2021-08-16T04:09:52.963951Z","shell.execute_reply.started":"2021-08-16T04:09:52.952837Z","shell.execute_reply":"2021-08-16T04:09:52.962989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = np.reshape(train, (len(train), 28, 28, 1))\ntest = np.reshape(test, (len(test), 28, 28, 1))\n\nautoencoder12 = create_deepautoencoder(train, 'adam', 'binary_crossentropy')\nautoencoder12.fit(train, train, epochs=50, batch_size=256, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:09:54.150710Z","iopub.execute_input":"2021-08-16T04:09:54.151085Z","iopub.status.idle":"2021-08-16T04:12:51.830155Z","shell.execute_reply.started":"2021-08-16T04:09:54.151056Z","shell.execute_reply":"2021-08-16T04:12:51.829157Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"autoencoder12.summary()","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:12:51.831932Z","iopub.execute_input":"2021-08-16T04:12:51.832637Z","iopub.status.idle":"2021-08-16T04:12:51.846507Z","shell.execute_reply.started":"2021-08-16T04:12:51.832584Z","shell.execute_reply":"2021-08-16T04:12:51.845346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoded12 = autoencoder12.predict(test)\ndisp_img(test)\ndisp_img(decoded12)","metadata":{"execution":{"iopub.status.busy":"2021-08-16T04:12:51.848728Z","iopub.execute_input":"2021-08-16T04:12:51.849309Z","iopub.status.idle":"2021-08-16T04:12:53.378886Z","shell.execute_reply.started":"2021-08-16T04:12:51.849265Z","shell.execute_reply":"2021-08-16T04:12:53.377685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In this case, the parameters of the optimizer were left at the default settings, but you may get different results if you adjust them.","metadata":{}}]}