{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nfrom tensorflow.keras.datasets import mnist\n\n(X_train, Y_train), (X_test, Y_test) = mnist.load_data()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import r2_score\nfrom sklearn.metrics import accuracy_score\n\nfrom tensorflow.keras.layers import Dense,Dropout,LSTM\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.losses import sparse_categorical_crossentropy, categorical_crossentropy, CategoricalCrossentropy, SparseCategoricalCrossentropy\nfrom tensorflow.keras.backend import sign\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras.utils import to_categorical","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('MNIST Dataset Shape:')\nprint('X_train: ' + str(X_train.shape))\nprint('Y_train: ' + str(Y_train.shape))\nprint('X_test:  '  + str(X_test.shape))\nprint('Y_test:  '  + str(Y_test.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train.astype('float32')\nX_test = X_test.astype('float32')\n# normalize the data\nX_train /= 255.0\nX_test /= 255.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# change input dimensions to 3 for LSTM input\nX_train = np.reshape(X_train, (60000, 28, 28))\nX_test = np.reshape(X_test, (10000, 28, 28))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_train = to_categorical(Y_train)\nY_test = to_categorical(Y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# LSTM model\nmodel = Sequential()\n\nmodel.add(LSTM(256, input_shape=(X_train.shape[1], 28)))\nmodel.add(Dropout(0.15))\nmodel.add(Dense(10, activation=\"sigmoid\"))\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"adam_modified = optimizers.Adam(learning_rate=0.005, beta_1=0.7, beta_2=0.9, amsgrad=False)\n\nmodel.compile(loss=\"categorical_crossentropy\", optimizer=adam_modified, metrics=[\"accuracy\"])\nmodel.fit(X_train, Y_train, epochs=5, validation_split=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = model.predict_classes(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"accuracy_score(np.argmax(Y_test, axis=1), predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def loss_curve(history):\n    \n    plt.figure(figsize=[8,6])\n    plt.plot(history.history['loss'],'r',linewidth=3.0)\n    plt.plot(history.history['val_loss'],'b',linewidth=3.0)\n    plt.legend(['Training loss', 'Validation Loss'],fontsize=18)\n    plt.xlabel('Epochs ',fontsize=16)\n    plt.ylabel('Loss',fontsize=16)\n    plt.title('Loss Curves',fontsize=16)\n\ndef accuracy_curve(history):\n    \n    plt.figure(figsize=[8,6])\n    plt.plot(history.history['accuracy'],'r',linewidth=3.0)\n    plt.plot(history.history['val_accuracy'],'b',linewidth=3.0)\n    plt.legend(['Training Accuracy', 'Validation Accuracy'],fontsize=18)\n    plt.xlabel('Epochs ',fontsize=16)\n    plt.ylabel('Accuracy',fontsize=16)\n    plt.title('Accuracy Curves',fontsize=16)\n    \naccuracy_curve(model.history)\nloss_curve(model.history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#we should review \ndef plot_predictions(x_test, y_true, y_prediction):\n    #indice = rd.choice(np[0])   random 10 values\n    num_cols = 5\n    num_rows = 2\n    fig, axes = plt.subplots(num_rows, num_cols, figsize=(1.5*num_cols,2*num_rows))\n    for i in range(len(x_test)):\n        ax = axes[i//num_cols, i%num_cols]\n        ax.imshow(x_test[i], cmap='gray_r')\n        ax.set_title('t: {} \\n p: {}'.format(np.argmax(y_true[i]), y_prediction[i]))\n    plt.tight_layout()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_digit(X,y):\n    visual = np.reshape(X[indice,:n_cols_train], (28, 28))\n    imgplot = plt.imshow(visual)\n   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_predictions(X_test[:10], Y_test[0:10], predictions[0:10])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_tf = tf.convert_to_tensor(X_test, dtype=tf.float32) #X_tf = tf.cast(X_test, tf.float32) #cast?\n# Y_tf = tf.convert_to_tensor(Y_test)\n\nwith tf.GradientTape() as tape:\n    tape.watch(X_tf) \n    lossFunction = categorical_crossentropy(Y_test, to_categorical(predictions))\ngradientFunction = tape.gradient(lossFunction, X_tf) # gradient function returns None\nprint(gradientFunction)\nsigned_grad = tf.sign(gradientFunction)\nperturbed_data = X_test + 0.3 * signed_grad #0.2\nX_perturbed = perturbed_data.numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def adversarial_pattern(image, label, predictions):\n    image = tf.cast(image, tf.float32)\n    \n    with tf.GradientTape() as tape:\n        tape.watch(image)\n        loss = tf.keras.losses.CategoricalCrossentropy(label, predictions)\n    \n    gradient = tape.gradient(loss, image)\n    \n    signed_grad = tf.sign(gradient)\n    \n    return signed_grad","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"perturbations = adversarial_pattern(X_test[0].reshape(1, 28, 28), Y_test[0], to_categorical(predictions[0]).numpy()\n\nperturbed_data = X_test + 0.3 * perturbations #0.2\nX_perturbed = perturbed_data.numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.executing_eagerly()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"adversarial_predictions = model.predict_classes(X_perturbed)\naccuracy_score(Y_test, adversarial_predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test_plot = X_test[:, :, 0]\nx_perturbed_plot = X_perturbed[:, :, 0]\nplt.figure(figsize=(20,2))\nplt.plot(x_test_plot[:50], color='blue',label='Actual data')\nplt.plot(x_perturbed_plot[:50], color='orange',label='Perturbated data')\nplt.xlabel('Sample number')\nplt.ylabel('Eye state')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"adversarial_predictions = model.predict_classes(X_perturbed)\nplot_predictions(Y_test[0:300], adversarial_predictions[0:300], \"Predictions made by LSTM model\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"accuracy_score(Y_test, adversarial_predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions[:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}