{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introdução","metadata":{}},{"cell_type":"markdown","source":"**Resumo:**\n\nO exemplo aqui desenvolvido tem como objetivo apresentar conceitos iniciais de implementação de redes neurais com python e tensorflow/keras. Esse modelo apresenta um modelo de Rede Neural Convolucional (CNN) básico que pode ser expandido mudando o número de neurônios e camadas. Em adaptações mais avançadas, pode-se estudar possibilidade de otimização de hyperparâmetros e outras técnincas como aumento de dados.\n\n**Não é objetivo nosso desenvolver e otimizar o modelo de classificação**. O exemplo tem objetivo meramente didático.\n\n---\n\n**Para saber mais:**\n* [Palestras e cursos do Ocean](http://www.oceanbrasil.com/)\n* François Chollet. Deep Learning with Python. Manning Publications, 2017.\n* Ian Goodfellow and Yoshua Bengio and Aaron Courville. [Deep Learning](https://www.deeplearningbook.org/). MIT Press, 2016.","metadata":{}},{"cell_type":"markdown","source":"# Bibliotecas e Dados","metadata":{}},{"cell_type":"code","source":"# Bibliotecas necessárias\n# Manipulação de dados\nimport pandas as pd\n# Redes Neurais\nfrom tensorflow import keras\nfrom tensorflow.keras.datasets import mnist\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, Conv2D, MaxPooling2D, Flatten\nfrom tensorflow.keras.optimizers import RMSprop\n# Plot\nimport matplotlib.pyplot as plt\n%matplotlib inline\n# Avaliação\nfrom sklearn.metrics import classification_report, confusion_matrix\n","metadata":{"ExecuteTime":{"end_time":"2019-01-16T20:39:41.500258Z","start_time":"2019-01-16T20:39:39.261856Z"},"execution":{"iopub.status.busy":"2021-09-18T12:53:36.930157Z","iopub.execute_input":"2021-09-18T12:53:36.930507Z","iopub.status.idle":"2021-09-18T12:53:36.942319Z","shell.execute_reply.started":"2021-09-18T12:53:36.930476Z","shell.execute_reply":"2021-09-18T12:53:36.940987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lendo o dataset Kaggle\ntrain = pd.read_csv(\"../input/digit-recognizer/train.csv\")\n\n#Alternativa ler do próprio keras\n#(x_train, y_train), (x_test, y_test) = mnist.load_data()\n","metadata":{"execution":{"iopub.status.busy":"2021-09-18T12:53:37.787319Z","iopub.execute_input":"2021-09-18T12:53:37.787730Z","iopub.status.idle":"2021-09-18T12:53:41.243940Z","shell.execute_reply.started":"2021-09-18T12:53:37.787682Z","shell.execute_reply":"2021-09-18T12:53:41.242965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Analisando o dataset\nprint(\"Quantidade de elementos de treino: {}\". format(len(train)))\nprint(train.head())","metadata":{"execution":{"iopub.status.busy":"2021-09-18T12:53:41.247660Z","iopub.execute_input":"2021-09-18T12:53:41.248393Z","iopub.status.idle":"2021-09-18T12:53:41.264403Z","shell.execute_reply.started":"2021-09-18T12:53:41.248343Z","shell.execute_reply":"2021-09-18T12:53:41.263079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Separando x_train e y_train\nY = train[\"label\"]\nX = train.drop(labels = [\"label\"],axis = 1)\nprint(X.head())\n# Em formato numpy array de imagens 28 x 28\n#x = X.values.reshape(-1,28,28,1)\n#print(x[0])","metadata":{"execution":{"iopub.status.busy":"2021-09-18T12:53:41.266303Z","iopub.execute_input":"2021-09-18T12:53:41.267049Z","iopub.status.idle":"2021-09-18T12:53:41.395434Z","shell.execute_reply.started":"2021-09-18T12:53:41.266981Z","shell.execute_reply":"2021-09-18T12:53:41.394166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X.shape)","metadata":{"execution":{"iopub.status.busy":"2021-09-18T12:53:41.397292Z","iopub.execute_input":"2021-09-18T12:53:41.397982Z","iopub.status.idle":"2021-09-18T12:53:41.408570Z","shell.execute_reply.started":"2021-09-18T12:53:41.397933Z","shell.execute_reply":"2021-09-18T12:53:41.406993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Numtendi nada!\n# Bora ver com matplotlib\nplt.imshow(X.values[0].reshape(28,28), cmap=plt.cm.binary)\nplt.show()\nprint('Label: {}'.format(Y[0]))","metadata":{"ExecuteTime":{"end_time":"2019-01-08T20:49:47.144492Z","start_time":"2019-01-08T20:49:47.016025Z"},"execution":{"iopub.status.busy":"2021-09-18T12:53:41.412614Z","iopub.execute_input":"2021-09-18T12:53:41.413120Z","iopub.status.idle":"2021-09-18T12:53:41.588102Z","shell.execute_reply.started":"2021-09-18T12:53:41.413060Z","shell.execute_reply":"2021-09-18T12:53:41.586881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Transformando a imagem 2d em um numpy array (imagem 28*28)\nx = X.values.reshape(42000, 28, 28, 1)\n\n#Normalizando para valores entre 0 e 1\nx = x.astype('float32')\nx /= 255\n\n#print(x[0])","metadata":{"ExecuteTime":{"end_time":"2019-01-08T20:49:57.425537Z","start_time":"2019-01-08T20:49:57.272377Z"},"execution":{"iopub.status.busy":"2021-09-18T12:53:41.590653Z","iopub.execute_input":"2021-09-18T12:53:41.591320Z","iopub.status.idle":"2021-09-18T12:53:41.664148Z","shell.execute_reply.started":"2021-09-18T12:53:41.591273Z","shell.execute_reply":"2021-09-18T12:53:41.662840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Vamos ajustar o formato da saida\nnum_classes = 10\n\n# Convertendo para um vetor de saida com 10 dimensoes\n# ex. 8 => [0,0,0,0,0,0,0,0,1,0]\ny = keras.utils.to_categorical(Y, num_classes)\nprint(y[0])","metadata":{"ExecuteTime":{"end_time":"2019-01-08T20:50:48.007428Z","start_time":"2019-01-08T20:50:47.986527Z"},"execution":{"iopub.status.busy":"2021-09-18T12:53:41.666368Z","iopub.execute_input":"2021-09-18T12:53:41.666947Z","iopub.status.idle":"2021-09-18T12:53:41.676764Z","shell.execute_reply.started":"2021-09-18T12:53:41.666866Z","shell.execute_reply":"2021-09-18T12:53:41.675515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Separando uma parte para treino (90%) e outra para validação (10%)\nfrom sklearn.model_selection import train_test_split\nx_train, x_val, y_train, y_val = train_test_split(x, y, test_size = 0.1, random_state=5)\nprint('Qtde de treino: {}'.format(len(x_train)))\nprint('Qtde de validação: {}'.format(len(x_val)))","metadata":{"execution":{"iopub.status.busy":"2021-09-18T12:53:41.679247Z","iopub.execute_input":"2021-09-18T12:53:41.679741Z","iopub.status.idle":"2021-09-18T12:53:42.424041Z","shell.execute_reply.started":"2021-09-18T12:53:41.679694Z","shell.execute_reply":"2021-09-18T12:53:42.422699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Criando e treinando o Modelo","metadata":{}},{"cell_type":"code","source":"# Criando o modelo Sequential\n# Sequential: Modelo Keras de ir adicionando camadas (como um lego)\n# Conv2D: Camada com kernels (filtros) que percorrem a imagem extraindo caracterísitcas (mapas de caracteísticas)\n# MaxPooling2D: Camada que reduz a dimensionalidade dos mapas de características 2D\n# Flatten: Camada que transforma um mapa de características 2D num vetor para classficador final\n# Dense: Camada onde todas as entradas estão conectadas em cada neurônio (totalmente conectada)\n# Dropout: Camada usa durante treino que descarta aleatoriamente um percentual de conexões (reduz overfitting)\n\nmodel = Sequential()\nmodel.add(Conv2D(20, kernel_size=(3, 3),\n                 activation='relu',\n                 input_shape=(28,28,1)))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Conv2D(40, kernel_size=(3,3), activation='relu'))\nmodel.add(Flatten())\nmodel.add(Dense(50, activation='relu'))\nmodel.add(Dropout(0.2))\nmodel.add(Dense(num_classes, activation='softmax'))\nmodel.summary()","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:38:00.233865Z","start_time":"2019-01-08T21:38:00.15079Z"},"scrolled":true,"execution":{"iopub.status.busy":"2021-09-18T12:53:42.425881Z","iopub.execute_input":"2021-09-18T12:53:42.426632Z","iopub.status.idle":"2021-09-18T12:53:42.517627Z","shell.execute_reply.started":"2021-09-18T12:53:42.426569Z","shell.execute_reply":"2021-09-18T12:53:42.516551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compila o modelo\nmodel.compile(loss='categorical_crossentropy',\n              optimizer=RMSprop(),\n              metrics=['accuracy'])","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:33:17.184644Z","start_time":"2019-01-08T21:33:17.129141Z"},"execution":{"iopub.status.busy":"2021-09-13T23:35:29.012085Z","iopub.execute_input":"2021-09-13T23:35:29.012399Z","iopub.status.idle":"2021-09-13T23:35:29.048088Z","shell.execute_reply.started":"2021-09-13T23:35:29.012363Z","shell.execute_reply":"2021-09-13T23:35:29.047264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Treina com os parte dos dados\nbatch_size = 32\nepochs = 20\n\n#Salvar o melhor modelo\ncallbacks_list = [\n    keras.callbacks.ModelCheckpoint(\n        filepath='model.h5',\n        monitor='val_loss', save_best_only=True, verbose=1),\n    keras.callbacks.EarlyStopping(monitor='val_loss', patience=10,verbose=1)\n]\n\nhistory = model.fit(x_train, y_train,\n                    batch_size=batch_size,\n                    epochs=epochs,\n                    callbacks = callbacks_list,\n                    verbose=1,\n                    validation_data=(x_val, y_val))","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:36:15.501168Z","start_time":"2019-01-08T21:33:19.322895Z"},"execution":{"iopub.status.busy":"2021-09-13T23:35:29.049321Z","iopub.execute_input":"2021-09-13T23:35:29.049674Z","iopub.status.idle":"2021-09-13T23:36:46.361421Z","shell.execute_reply.started":"2021-09-13T23:35:29.049638Z","shell.execute_reply":"2021-09-13T23:36:46.360418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Vamos ver como foi o treino?\n\nfig, ax = plt.subplots(1,2, figsize=(16,8))\nax[0].plot(history.history['loss'], color='b', label=\"Training loss\")\nax[0].plot(history.history['val_loss'], color='r', label=\"validation loss\",axes =ax[0])\nlegend = ax[0].legend(loc='best', shadow=True)\n\nax[1].plot(history.history['accuracy'], color='b', label=\"Training accuracy\")\nax[1].plot(history.history['val_accuracy'], color='r',label=\"Validation accuracy\")\nlegend = ax[1].legend(loc='best', shadow=True)","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:36:20.243921Z","start_time":"2019-01-08T21:36:20.064201Z"},"execution":{"iopub.status.busy":"2021-09-13T23:36:46.365642Z","iopub.execute_input":"2021-09-13T23:36:46.365915Z","iopub.status.idle":"2021-09-13T23:36:46.666109Z","shell.execute_reply.started":"2021-09-13T23:36:46.365881Z","shell.execute_reply":"2021-09-13T23:36:46.665074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.models import load_model\n# Load the best saved model\nmodel = load_model('model.h5')","metadata":{"execution":{"iopub.status.busy":"2021-09-13T23:36:46.667582Z","iopub.execute_input":"2021-09-13T23:36:46.667947Z","iopub.status.idle":"2021-09-13T23:36:47.108005Z","shell.execute_reply.started":"2021-09-13T23:36:46.667909Z","shell.execute_reply":"2021-09-13T23:36:47.107145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Testa\nscore = model.evaluate(x_val, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:36:39.779842Z","start_time":"2019-01-08T21:36:38.946173Z"},"execution":{"iopub.status.busy":"2021-09-13T23:36:47.109247Z","iopub.execute_input":"2021-09-13T23:36:47.109611Z","iopub.status.idle":"2021-09-13T23:36:47.501623Z","shell.execute_reply.started":"2021-09-13T23:36:47.109575Z","shell.execute_reply":"2021-09-13T23:36:47.500659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Testando uma entrada qualquer\nprint(y_train[10])\nprint(model.predict(x_train[10].reshape((1,28,28,1))))\nprint(model.predict_classes(x_train[10].reshape((1,28,28,1))))","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:06:04.363715Z","start_time":"2019-01-08T21:06:04.27849Z"},"execution":{"iopub.status.busy":"2021-09-13T23:36:47.503217Z","iopub.execute_input":"2021-09-13T23:36:47.503978Z","iopub.status.idle":"2021-09-13T23:36:47.605725Z","shell.execute_reply.started":"2021-09-13T23:36:47.503936Z","shell.execute_reply":"2021-09-13T23:36:47.603185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Avaliando o Modelo","metadata":{}},{"cell_type":"code","source":"import itertools\n\n#Plot the confusion matrix. Set Normalize = True/False\ndef plot_confusion_matrix(cm, classes, normalize=True, title='Confusion matrix', cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.figure(figsize=(10,10))\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        cm = np.around(cm, decimals=2)\n        cm[np.isnan(cm)] = 0.0\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')","metadata":{"execution":{"iopub.status.busy":"2021-09-13T23:36:47.607159Z","iopub.execute_input":"2021-09-13T23:36:47.607728Z","iopub.status.idle":"2021-09-13T23:36:47.620739Z","shell.execute_reply.started":"2021-09-13T23:36:47.607685Z","shell.execute_reply":"2021-09-13T23:36:47.619767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Vendo alguns reports# Vendo alguns reports\n# Usando sklearn\nimport numpy as np\n\n# Classificando toda base de teste\ny_pred = model.predict_classes(x_val)\n# voltando pro formato de classes\ny_test_c = np.argmax(y_val, axis=1)\ntarget_names = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']\n\n#Confution Matrix\ncm = confusion_matrix(y_test_c, y_pred)\nplot_confusion_matrix(cm, target_names, normalize=False, title='Confusion Matrix')\n\nprint('Classification Report')\nprint(classification_report(y_test_c, y_pred, target_names=target_names))","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:24:57.7654Z","start_time":"2019-01-08T21:24:57.220867Z"},"execution":{"iopub.status.busy":"2021-09-13T23:36:47.622183Z","iopub.execute_input":"2021-09-13T23:36:47.622734Z","iopub.status.idle":"2021-09-13T23:36:48.325434Z","shell.execute_reply.started":"2021-09-13T23:36:47.622691Z","shell.execute_reply":"2021-09-13T23:36:48.324405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Gerando Saída","metadata":{}},{"cell_type":"code","source":"# Gerando saída para dataset de teste\n\n#Carrega dataset de teste\ntest = pd.read_csv(\"../input/digit-recognizer/test.csv\")\nprint(\"Qtde de testes: {}\".format(len(test)))\n# Bota no formato numpy e normaliza\nx_test = test.values.reshape(len(test),28,28,1)\nx_test = x_test.astype('float32')\nx_test /= 255\n\n# Faz classificação para dataset de teste\ny_pred = model.predict_classes(x_test)\n\n# Verficando algum exemplo\ni = 0\nplt.imshow(test.values[i].reshape(28,28), cmap=plt.cm.binary)\nplt.show()\nprint('Previsto: {}'.format(y_pred[i]))\n\n# Botando no formato de saída (competição Kaggle)\nresults = pd.Series(y_pred,name=\"Label\")\nsubmission = pd.concat([pd.Series(range(1,len(y_pred)+1),name = \"ImageId\"),results],axis = 1)\nprint(submission.head(10))\n#Salvando Arquivo\nsubmission.to_csv(\"mlp_mnist_v1.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-13T23:36:48.326901Z","iopub.execute_input":"2021-09-13T23:36:48.327231Z","iopub.status.idle":"2021-09-13T23:36:52.188446Z","shell.execute_reply.started":"2021-09-13T23:36:48.327194Z","shell.execute_reply":"2021-09-13T23:36:52.187761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Teste Adicional: Com ruído","metadata":{}},{"cell_type":"code","source":"#introduzindo ruido\nimport numpy as np\nmean = 0.\nstddev = 0.2\nnoise = np.random.normal(mean, stddev, (4200, 28, 28,1))\nx_te = x_val + noise\nx_te = np.clip(x_te, 0., 1.)\n\nplt.imshow(x_te.reshape(4200, 28,28)[0], cmap=plt.cm.binary)\nplt.show()\n\n# Testa\nscore = model.evaluate(x_te, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])","metadata":{"execution":{"iopub.status.busy":"2021-09-13T23:36:52.189631Z","iopub.execute_input":"2021-09-13T23:36:52.189987Z","iopub.status.idle":"2021-09-13T23:36:52.852619Z","shell.execute_reply.started":"2021-09-13T23:36:52.189955Z","shell.execute_reply":"2021-09-13T23:36:52.851658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Com ruído, percebemos que a acurácia dos exemplos de validação caiu, mas muito menos que o modelo MLP clássico (ver exemplo de MLP).\nUm modelo Convolucional (CNN) captura melhor regiões, ou padrões espaciais de pixels, tornando o modelo mais robusto.","metadata":{}},{"cell_type":"markdown","source":"# Atividade 2: Experimentação para melhoria do resultado do modelo\nAluno: Bruno Mendes","metadata":{}},{"cell_type":"code","source":"# Fixa a semente aleatória para garantir consistência dos resultados entre execuções\nfrom numpy.random import seed\nseed(33)\nimport tensorflow as tf\ntf.random.set_seed(33)","metadata":{"execution":{"iopub.status.busy":"2021-09-13T23:36:52.854186Z","iopub.execute_input":"2021-09-13T23:36:52.854835Z","iopub.status.idle":"2021-09-13T23:36:52.867006Z","shell.execute_reply.started":"2021-09-13T23:36:52.854795Z","shell.execute_reply":"2021-09-13T23:36:52.866097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Criando o modelo Sequential\n# Sequential: Modelo Keras de ir adicionando camadas (como um lego)\n# Conv2D: Camada com kernels (filtros) que percorrem a imagem extraindo caracterísitcas (mapas de caracteísticas)\n# MaxPooling2D: Camada que reduz a dimensionalidade dos mapas de características 2D\n# Flatten: Camada que transforma um mapa de características 2D num vetor para classficador final\n# Dense: Camada onde todas as entradas estão conectadas em cada neurônio (totalmente conectada)\n# Dropout: Camada usa durante treino que descarta aleatoriamente um percentual de conexões (reduz overfitting)\n\nmodel = Sequential()\nmodel.add(Conv2D(20, kernel_size=(3, 3),\n                 activation='relu',\n                 input_shape=(28,28,1)))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Conv2D(40, kernel_size=(3,3), activation='relu'))\nmodel.add(Flatten())\nmodel.add(Dense(50, activation='relu'))\nmodel.add(Dropout(0.2))\nmodel.add(Dense(num_classes, activation='softmax'))\nmodel.summary()\n\n# Compila o modelo\nmodel.compile(loss='categorical_crossentropy',\n              optimizer=RMSprop(),\n              metrics=['accuracy', 'AUC'])\n\n# Treina com os parte dos dados\nbatch_size = 32\nepochs = 20\n\n#Salvar o melhor modelo\ncallbacks_list = [\n    keras.callbacks.ModelCheckpoint(\n        filepath='model.bruno',\n        monitor='val_loss', save_best_only=True, verbose=1),\n    keras.callbacks.EarlyStopping(monitor='val_loss', patience=0,verbose=1)\n]\n\nhistory = model.fit(x_train, y_train,\n                    batch_size=batch_size,\n                    epochs=epochs,\n                    callbacks = callbacks_list,\n                    verbose=1,\n                    validation_data=(x_val, y_val))\n\n# Testa\nscore = model.evaluate(x_val, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])\nprint('Test ROC AUC:', score[2])","metadata":{"execution":{"iopub.status.busy":"2021-09-13T23:36:52.870849Z","iopub.execute_input":"2021-09-13T23:36:52.871267Z","iopub.status.idle":"2021-09-13T23:37:39.569841Z","shell.execute_reply.started":"2021-09-13T23:36:52.871172Z","shell.execute_reply":"2021-09-13T23:37:39.566651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 32\nMAX_EPOCHS = 20\nMODEL_ID = \"model.bruno2\"\n\n# Baseado, em: https://www.kaggle.com/takahiroyoshida012/cnn-tensorflow-keras-vs-randomforest-acc-99\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\nfrom tensorflow.keras.optimizers import Adam, Adagrad, RMSprop, SGD\n\nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 56, \n                 kernel_size = 7, \n                 activation ='relu', \n                 input_shape = (28,28,1)))\nmodel.add(Conv2D(filters = 56, \n                 kernel_size = 7, \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2)))\nmodel.add(Dropout(0.1))\n\n\nmodel.add(Conv2D(filters = 112, \n                 kernel_size = 3,\n                 activation ='relu'))\nmodel.add(Conv2D(filters = 112, \n                 kernel_size = 3,\n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2)))\nmodel.add(Dropout(0.1))\n\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = \"relu\"))\nmodel.add(Dropout(0.2))\n\nmodel.add(Dense(num_classes, activation = \"softmax\"))\n\n# Compila o modelo\nmodel.compile(loss='categorical_crossentropy',\n              optimizer=RMSprop(),\n              metrics=['accuracy', 'AUC'])\n\n# Salva o melhor modelo\ncb_save_best_model = keras.callbacks.ModelCheckpoint(filepath=MODEL_ID,\n                                                           monitor='val_loss', \n                                                           save_best_only=True, \n                                                           verbose=1)\n\n# Encerra o treino antecipadamente se não houver evolução\ncb_early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', \n                                              patience=2,\n                                              verbose=1)\n\nhistory = model.fit(x_train, y_train,\n                    batch_size=BATCH_SIZE,\n                    epochs=MAX_EPOCHS,\n                    callbacks = [\n                        cb_save_best_model,\n                        cb_early_stop],\n                    verbose=1,\n                    validation_data=(x_val, y_val))\n\n# Carrega o melhor modelo\nmodel = load_model(MODEL_ID)\n\n# Avalia o melhor modelo\nscore = model.evaluate(x_val, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])\nprint('Test ROC AUC:', score[2])","metadata":{"execution":{"iopub.status.busy":"2021-09-13T23:37:39.574107Z","iopub.execute_input":"2021-09-13T23:37:39.574353Z","iopub.status.idle":"2021-09-13T23:38:05.905931Z","shell.execute_reply.started":"2021-09-13T23:37:39.574328Z","shell.execute_reply":"2021-09-13T23:38:05.905209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 100\nMAX_EPOCHS = 20\nMODEL_ID = \"model.bruno3\"\n\n# Baseado, em: https://www.kaggle.com/takahiroyoshida012/cnn-tensorflow-keras-vs-randomforest-acc-99\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\nfrom tensorflow.keras.optimizers import Adam, Adagrad, RMSprop, SGD\n\nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 56, \n                 kernel_size = 7, \n                 activation ='relu', \n                 input_shape = (28,28,1)))\nmodel.add(Conv2D(filters = 56, \n                 kernel_size = 7, \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.2))\n\n\nmodel.add(Conv2D(filters = 112, \n                 kernel_size = 3,\n                 activation ='relu'))\nmodel.add(Conv2D(filters = 112, \n                 kernel_size = 3,\n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.2))\n\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = \"relu\"))\nmodel.add(Dropout(0.2))\n\nmodel.add(Dense(num_classes, activation = \"softmax\"))\n\n# Compila o modelo\nmodel.compile(loss='categorical_crossentropy',\n              optimizer=RMSprop(),\n              metrics=['accuracy', 'AUC'])\n\n# Salva o melhor modelo\ncb_save_best_model = keras.callbacks.ModelCheckpoint(filepath=MODEL_ID,\n                                                           monitor='val_loss', \n                                                           save_best_only=True, \n                                                           verbose=1)\n\n# Encerra o treino antecipadamente se não houver evolução\ncb_early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', \n                                              patience=4,\n                                              verbose=1)\n\nhistory = model.fit(x_train, y_train,\n                    batch_size=BATCH_SIZE,\n                    epochs=MAX_EPOCHS,\n                    callbacks = [\n                        cb_save_best_model,\n                        cb_early_stop],\n                    verbose=1,\n                    validation_data=(x_val, y_val))\n\n# Carrega o melhor modelo\nmodel = load_model(MODEL_ID)\n\n# Avalia o melhor modelo\nscore = model.evaluate(x_val, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])\nprint('Test ROC AUC:', score[2])","metadata":{"execution":{"iopub.status.busy":"2021-09-13T23:38:05.90713Z","iopub.execute_input":"2021-09-13T23:38:05.907385Z","iopub.status.idle":"2021-09-13T23:38:26.231662Z","shell.execute_reply.started":"2021-09-13T23:38:05.907359Z","shell.execute_reply":"2021-09-13T23:38:26.230745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 100\nMAX_EPOCHS = 20\nMODEL_ID = \"model.bruno4\"\n\n# Baseado, em: https://www.kaggle.com/takahiroyoshida012/cnn-tensorflow-keras-vs-randomforest-acc-99\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, AvgPool2D\nfrom tensorflow.keras.optimizers import Adam, Adagrad, RMSprop, SGD\n\nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 112, \n                 kernel_size = 7, \n                 activation ='relu', \n                 input_shape = (28,28,1)))\nmodel.add(Conv2D(filters = 112, \n                 kernel_size = 7, \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.2))\n\n\nmodel.add(Conv2D(filters = 56, \n                 kernel_size = 3,\n                 activation ='relu'))\nmodel.add(Conv2D(filters = 56, \n                 kernel_size = 3,\n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.2))\n\n\nmodel.add(Flatten())\nmodel.add(Dense(128, activation = \"relu\"))\nmodel.add(Dropout(0.2))\n\nmodel.add(Dense(num_classes, activation = \"softmax\"))\n\n# Compila o modelo\nmodel.compile(loss='categorical_crossentropy',\n              optimizer='adam',\n              metrics=['accuracy', 'AUC'])\n\n# Salva o melhor modelo\ncb_save_best_model = keras.callbacks.ModelCheckpoint(filepath=MODEL_ID,\n                                                     monitor='val_loss', \n                                                     save_best_only=True, \n                                                     verbose=1)\n\n# Encerra o treino antecipadamente se não houver evolução\ncb_early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', \n                                              patience=2,\n                                              verbose=1)\n\nhistory = model.fit(x_train, y_train,\n                    batch_size=BATCH_SIZE,\n                    epochs=MAX_EPOCHS,\n                    callbacks = [\n                        cb_save_best_model,\n                        cb_early_stop],\n                    validation_data=(x_val, y_val),\n                    verbose=1)\n\n# Carrega o melhor modelo\nmodel = load_model(MODEL_ID)\n\n# Avalia o melhor modelo\nscore = model.evaluate(x_val, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])\nprint('Test ROC AUC:', score[2])","metadata":{"execution":{"iopub.status.busy":"2021-09-13T23:38:26.233173Z","iopub.execute_input":"2021-09-13T23:38:26.233523Z","iopub.status.idle":"2021-09-13T23:39:02.664962Z","shell.execute_reply.started":"2021-09-13T23:38:26.233486Z","shell.execute_reply":"2021-09-13T23:39:02.664071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 100\nMAX_EPOCHS = 20\nMODEL_ID = \"model.bruno5\"\n\n# Baseado, em: https://www.kaggle.com/takahiroyoshida012/cnn-tensorflow-keras-vs-randomforest-acc-99\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, AvgPool2D\nfrom tensorflow.keras.optimizers import Adam, Adagrad, RMSprop, SGD\n\nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 112, \n                 kernel_size = 7, \n                 activation ='relu', \n                 input_shape = (28,28,1)))\nmodel.add(Conv2D(filters = 112, \n                 kernel_size = 7, \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.1))\n\n\nmodel.add(Conv2D(filters = 56, \n                 kernel_size = 3,\n                 activation ='relu'))\nmodel.add(Conv2D(filters = 56, \n                 kernel_size = 3,\n                 activation ='relu'))\nmodel.add(AvgPool2D(pool_size=2))\nmodel.add(Dropout(0.2))\n\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = \"relu\"))\nmodel.add(Dropout(0.25))\n\nmodel.add(Dense(num_classes, activation = \"softmax\"))\n\n# Compila o modelo\nmodel.compile(loss='categorical_crossentropy',\n              optimizer='adam',\n              metrics=['accuracy', 'AUC'])\n\n# Salva o melhor modelo\ncb_save_best_model = keras.callbacks.ModelCheckpoint(filepath=MODEL_ID,\n                                                     monitor='val_loss', \n                                                     save_best_only=True, \n                                                     verbose=1)\n\n# Encerra o treino antecipadamente se não houver evolução\ncb_early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', \n                                              patience=2,\n                                              verbose=1)\n\nhistory = model.fit(x_train, y_train,\n                    batch_size=BATCH_SIZE,\n                    epochs=MAX_EPOCHS,\n                    callbacks = [\n                        cb_save_best_model,\n                        cb_early_stop],\n                    validation_data=(x_val, y_val),\n                    verbose=1)\n\n# Carrega o melhor modelo\nmodel = load_model(MODEL_ID)\n\n# Avalia o melhor modelo\nscore = model.evaluate(x_val, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])\nprint('Test ROC AUC:', score[2])","metadata":{"execution":{"iopub.status.busy":"2021-09-13T23:39:02.667585Z","iopub.execute_input":"2021-09-13T23:39:02.66794Z","iopub.status.idle":"2021-09-13T23:39:29.27815Z","shell.execute_reply.started":"2021-09-13T23:39:02.667902Z","shell.execute_reply":"2021-09-13T23:39:29.277188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-14T00:05:11.391073Z","iopub.execute_input":"2021-09-14T00:05:11.391399Z","iopub.status.idle":"2021-09-14T00:05:11.400972Z","shell.execute_reply.started":"2021-09-14T00:05:11.39137Z","shell.execute_reply":"2021-09-14T00:05:11.40006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 100\nMAX_EPOCHS = 20\nMODEL_ID = \"model.bruno6\"\n\n# Baseado, em: https://www.kaggle.com/takahiroyoshida012/cnn-tensorflow-keras-vs-randomforest-acc-99\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, AvgPool2D\nfrom tensorflow.keras.optimizers import Adam, Adagrad, RMSprop, SGD\n\nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 5, \n                 kernel_size = 3, \n                 activation ='relu', \n                 input_shape = (28,28,1)))\nmodel.add(Conv2D(filters = 5, \n                 kernel_size = 3, \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.1))\n\n\nmodel.add(Conv2D(filters = 10, \n                 kernel_size = 3,\n                 activation ='relu'))\nmodel.add(Conv2D(filters = 10, \n                 kernel_size = 3,\n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.1))\n\n\nmodel.add(Conv2D(filters = 20, \n                 kernel_size = 2,\n                 activation ='relu'))\nmodel.add(Conv2D(filters = 20, \n                 kernel_size = 2,\n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.1))\n\nmodel.add(Flatten())\n\nmodel.add(Dense(100, activation = \"relu\"))\nmodel.add(Dropout(0.1))\nmodel.add(Dense(50, activation = \"relu\"))\nmodel.add(Dropout(0.1))\nmodel.add(Dense(20, activation = \"relu\"))\nmodel.add(Dropout(0.1))\n\nmodel.add(Dense(num_classes, activation = \"softmax\"))\n\nmodel.summary()\n\n# Compila o modelo\nmodel.compile(loss='categorical_crossentropy',\n              optimizer='adam',\n              metrics=['accuracy', 'AUC'])\n\n# Salva o melhor modelo\ncb_save_best_model = keras.callbacks.ModelCheckpoint(filepath=MODEL_ID,\n                                                     monitor='val_loss', \n                                                     save_best_only=True, \n                                                     verbose=1)\n\n# Encerra o treino antecipadamente se não houver evolução\ncb_early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', \n                                              patience=2,\n                                              verbose=1)\n\nhistory = model.fit(x_train, y_train,\n                    batch_size=BATCH_SIZE,\n                    epochs=MAX_EPOCHS,\n                    callbacks = [\n                        cb_save_best_model,\n                        cb_early_stop],\n                    validation_data=(x_val, y_val),\n                    verbose=1)\n\n# Carrega o melhor modelo\nmodel = load_model(MODEL_ID)\n\n# Avalia o melhor modelo\nscore = model.evaluate(x_val, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])\nprint('Test ROC AUC:', score[2])","metadata":{"execution":{"iopub.status.busy":"2021-09-14T00:02:06.902681Z","iopub.execute_input":"2021-09-14T00:02:06.903023Z","iopub.status.idle":"2021-09-14T00:02:46.42047Z","shell.execute_reply.started":"2021-09-14T00:02:06.902994Z","shell.execute_reply":"2021-09-14T00:02:46.418361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 100\nMAX_EPOCHS = 30\nMODEL_ID = \"model.bruno7\"\n\n# Baseado, em: https://www.kaggle.com/takahiroyoshida012/cnn-tensorflow-keras-vs-randomforest-acc-99\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, AvgPool2D\nfrom tensorflow.keras.optimizers import Adam, Adagrad, RMSprop, SGD\n\nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 10, \n                 kernel_size = 3, \n                 activation ='relu', \n                 input_shape = (28,28,1)))\nmodel.add(Conv2D(filters = 10, \n                 kernel_size = 3, \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.1))\n\n\nmodel.add(Conv2D(filters = 20, \n                 kernel_size = 2,\n                 activation ='relu'))\nmodel.add(Conv2D(filters = 20, \n                 kernel_size = 2,\n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.1))\n\nmodel.add(Conv2D(filters = 20, \n                 kernel_size = 2,\n                 activation ='relu'))\nmodel.add(Conv2D(filters = 20, \n                 kernel_size = 2,\n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.1))\n\nmodel.add(Flatten())\n\nmodel.add(Dense(60, activation = \"relu\"))\nmodel.add(Dropout(0.1))\nmodel.add(Dense(20, activation = \"relu\"))\nmodel.add(Dropout(0.1))\n\nmodel.add(Dense(num_classes, activation = \"softmax\"))\n\nmodel.summary()\n\n# Compila o modelo\nmodel.compile(loss='categorical_crossentropy',\n              optimizer='adam',\n              metrics=['accuracy', 'AUC'])\n\n# Salva o melhor modelo\ncb_save_best_model = keras.callbacks.ModelCheckpoint(filepath=MODEL_ID,\n                                                     monitor='val_loss', \n                                                     save_best_only=True, \n                                                     verbose=1)\n\n# Encerra o treino antecipadamente se não houver evolução\ncb_early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', \n                                              patience=4,\n                                              verbose=1)\n\nhistory = model.fit(x_train, y_train,\n                    batch_size=BATCH_SIZE,\n                    epochs=MAX_EPOCHS,\n                    callbacks = [\n                        cb_save_best_model,\n                        cb_early_stop],\n                    validation_data=(x_val, y_val),\n                    verbose=1)\n\n# Carrega o melhor modelo\nmodel = load_model(MODEL_ID)\n\n# Avalia o melhor modelo\nscore = model.evaluate(x_val, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])\nprint('Test ROC AUC:', score[2])","metadata":{"execution":{"iopub.status.busy":"2021-09-14T00:16:11.758194Z","iopub.execute_input":"2021-09-14T00:16:11.758509Z","iopub.status.idle":"2021-09-14T00:16:51.985034Z","shell.execute_reply.started":"2021-09-14T00:16:11.75848Z","shell.execute_reply":"2021-09-14T00:16:51.983329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 100\nMAX_EPOCHS = 30\nMODEL_ID = \"model.bruno8\"\n\n# Baseado, em: https://www.kaggle.com/takahiroyoshida012/cnn-tensorflow-keras-vs-randomforest-acc-99\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, AvgPool2D\nfrom tensorflow.keras.optimizers import Adam, Adagrad, RMSprop, SGD\n\nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 10, \n                 kernel_size = 3, \n                 activation ='relu', \n                 input_shape = (28,28,1)))\nmodel.add(Conv2D(filters = 10, \n                 kernel_size = 3, \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.1))\n\n\nmodel.add(Conv2D(filters = 20, \n                 kernel_size = 2,\n                 activation ='relu'))\nmodel.add(Conv2D(filters = 20, \n                 kernel_size = 2,\n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.1))\n\nmodel.add(Conv2D(filters = 40, \n                 kernel_size = 1,\n                 activation ='elu'))\nmodel.add(Conv2D(filters = 40, \n                 kernel_size = 1,\n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.1))\n\nmodel.add(Flatten())\n\nmodel.add(Dense(40, activation = \"relu\"))\nmodel.add(Dropout(0.1))\nmodel.add(Dense(20, activation = \"relu\"))\nmodel.add(Dropout(0.1))\nmodel.add(Dense(10, activation = \"relu\"))\nmodel.add(Dropout(0.1))\n\nmodel.add(Dense(num_classes, activation = \"softmax\"))\n\nmodel.summary()\n\n# Compila o modelo\nmodel.compile(loss='categorical_crossentropy',\n              optimizer='adam',\n              metrics=['accuracy', 'AUC'])\n\n# Salva o melhor modelo\ncb_save_best_model = keras.callbacks.ModelCheckpoint(filepath=MODEL_ID,\n                                                     monitor='val_loss', \n                                                     save_best_only=True, \n                                                     verbose=1)\n\n# Encerra o treino antecipadamente se não houver evolução\ncb_early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', \n                                              patience=4,\n                                              verbose=1)\n\nhistory = model.fit(x_train, y_train,\n                    batch_size=BATCH_SIZE,\n                    epochs=MAX_EPOCHS,\n                    callbacks = [\n                        cb_save_best_model,\n                        cb_early_stop],\n                    validation_data=(x_val, y_val),\n                    verbose=1)\n\n# Carrega o melhor modelo\nmodel = load_model(MODEL_ID)\n\n# Avalia o melhor modelo\nscore = model.evaluate(x_val, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])\nprint('Test ROC AUC:', score[2])","metadata":{"execution":{"iopub.status.busy":"2021-09-14T00:20:07.893089Z","iopub.execute_input":"2021-09-14T00:20:07.8934Z","iopub.status.idle":"2021-09-14T00:21:35.03144Z","shell.execute_reply.started":"2021-09-14T00:20:07.893371Z","shell.execute_reply":"2021-09-14T00:21:35.030328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 100\nMAX_EPOCHS = 30\nMODEL_ID = \"model.bruno9\"\n\n# Baseado, em: https://www.kaggle.com/takahiroyoshida012/cnn-tensorflow-keras-vs-randomforest-acc-99\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, AvgPool2D\nfrom tensorflow.keras.optimizers import Adam, Adagrad, RMSprop, SGD\n\nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 10, \n                 kernel_size = 3, \n                 activation ='elu', \n                 input_shape = (28,28,1)))\nmodel.add(Conv2D(filters = 10, \n                 kernel_size = 3, \n                 activation ='elu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.1))\n\n\nmodel.add(Conv2D(filters = 20, \n                 kernel_size = 2,\n                 activation ='elu'))\nmodel.add(Conv2D(filters = 20, \n                 kernel_size = 2,\n                 activation ='elu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.1))\n\nmodel.add(Conv2D(filters = 40, \n                 kernel_size = 1,\n                 activation ='elu'))\nmodel.add(Conv2D(filters = 40, \n                 kernel_size = 1,\n                 activation ='elu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.1))\n\nmodel.add(Flatten())\n\nmodel.add(Dense(40, activation = \"elu\"))\nmodel.add(Dropout(0.1))\nmodel.add(Dense(20, activation = \"elu\"))\nmodel.add(Dropout(0.1))\nmodel.add(Dense(10, activation = \"elu\"))\nmodel.add(Dropout(0.1))\n\nmodel.add(Dense(num_classes, activation = \"softmax\"))\n\nmodel.summary()\n\n# Compila o modelo\nmodel.compile(loss='categorical_crossentropy',\n              optimizer='adam',\n              metrics=['accuracy', 'AUC'])\n\n# Salva o melhor modelo\ncb_save_best_model = keras.callbacks.ModelCheckpoint(filepath=MODEL_ID,\n                                                     monitor='val_loss', \n                                                     save_best_only=True, \n                                                     verbose=1)\n\n# Encerra o treino antecipadamente se não houver evolução\ncb_early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', \n                                              patience=4,\n                                              verbose=1)\n\nhistory = model.fit(x_train, y_train,\n                    batch_size=BATCH_SIZE,\n                    epochs=MAX_EPOCHS,\n                    callbacks = [\n                        cb_save_best_model,\n                        cb_early_stop],\n                    validation_data=(x_val, y_val),\n                    verbose=1)\n\n# Carrega o melhor modelo\nmodel = load_model(MODEL_ID)\n\n# Avalia o melhor modelo\nscore = model.evaluate(x_val, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])\nprint('Test ROC AUC:', score[2])","metadata":{"execution":{"iopub.status.busy":"2021-09-14T00:26:16.411332Z","iopub.execute_input":"2021-09-14T00:26:16.411686Z","iopub.status.idle":"2021-09-14T00:27:34.659227Z","shell.execute_reply.started":"2021-09-14T00:26:16.411656Z","shell.execute_reply":"2021-09-14T00:27:34.658391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 100\nMAX_EPOCHS = 20\nMODEL_ID = \"model.bruno10\"\n\n# Baseado, em: https://www.kaggle.com/takahiroyoshida012/cnn-tensorflow-keras-vs-randomforest-acc-99\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, AvgPool2D\nfrom tensorflow.keras.optimizers import Adam, Adagrad, RMSprop, SGD\nfrom tensorflow.keras.models import load_model\n\nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 56, \n                 kernel_size = 7, \n                 activation ='relu', \n                 input_shape = (28,28,1)))\nmodel.add(Conv2D(filters = 56, \n                 kernel_size = 7, \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.2))\n\nmodel.add(Conv2D(filters = 112, \n                 kernel_size = 3, \n                 activation ='relu'))\nmodel.add(Conv2D(filters = 112, \n                 kernel_size = 3, \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.2))\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = \"sigmoid\"))\nmodel.add(Dropout(0.2))\n\nmodel.add(Dense(num_classes, activation = \"softmax\"))\n\n# Compila o modelo\nmodel.compile(loss='categorical_crossentropy',\n              optimizer=\"adam\",\n              metrics=['accuracy', 'AUC'])\n\nmodel.summary()\n\n# Salva o melhor modelo\ncb_save_best_model = keras.callbacks.ModelCheckpoint(filepath=MODEL_ID,\n                                                           monitor='val_loss', \n                                                           save_best_only=True, \n                                                           verbose=1)\n\n# Encerra o treino antecipadamente se não houver evolução\ncb_early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', \n                                              patience=4,\n                                              verbose=1)\n\nhistory = model.fit(x_train, y_train,\n                    batch_size=BATCH_SIZE,\n                    epochs=MAX_EPOCHS,\n                    callbacks = [\n                        cb_save_best_model,\n                        cb_early_stop],\n                    verbose=1,\n                    validation_data=(x_val, y_val))\n\n# Carrega o melhor modelo\nmodel = load_model(MODEL_ID)\n\n# Avalia o melhor modelo\nscore = model.evaluate(x_val, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])\nprint('Test ROC AUC:', score[2])","metadata":{"execution":{"iopub.status.busy":"2021-09-18T12:57:52.340517Z","iopub.execute_input":"2021-09-18T12:57:52.341026Z","iopub.status.idle":"2021-09-18T12:58:41.654848Z","shell.execute_reply.started":"2021-09-18T12:57:52.340991Z","shell.execute_reply":"2021-09-18T12:58:41.653524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 50\nMAX_EPOCHS = 30\nMODEL_ID = \"model.bruno11\"\n\nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 100, \n                 kernel_size = 7, \n                 activation ='relu', \n                 input_shape = (28,28,1)))\nmodel.add(Conv2D(filters = 100, \n                 kernel_size = 7, \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.2))\n\nmodel.add(Conv2D(filters = 200, \n                 kernel_size = 3, \n                 activation ='relu'))\nmodel.add(Conv2D(filters = 200, \n                 kernel_size = 3, \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=2))\nmodel.add(Dropout(0.2))\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = \"sigmoid\"))\nmodel.add(Dropout(0.2))\n\nmodel.add(Dense(num_classes, activation = \"softmax\"))\n\n# Compila o modelo\nmodel.compile(loss='categorical_crossentropy',\n              optimizer=\"adam\",\n              metrics=['accuracy', 'AUC'])\n\n# Salva o melhor modelo\ncb_save_best_model = keras.callbacks.ModelCheckpoint(filepath=MODEL_ID,\n                                                           monitor='val_loss', \n                                                           save_best_only=True, \n                                                           verbose=1)\n\n# Encerra o treino antecipadamente se não houver evolução\ncb_early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', \n                                              patience=4,\n                                              verbose=1)\n\nhistory = model.fit(x_train, y_train,\n                    batch_size=BATCH_SIZE,\n                    epochs=MAX_EPOCHS,\n                    callbacks = [\n                        cb_save_best_model,\n                        cb_early_stop],\n                    verbose=1,\n                    validation_data=(x_val, y_val))\n\n# Carrega o melhor modelo\nmodel = load_model(MODEL_ID)\n\n# Avalia o melhor modelo\nscore = model.evaluate(x_val, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])\nprint('Test ROC AUC:', score[2])","metadata":{"execution":{"iopub.status.busy":"2021-09-14T01:16:45.452445Z","iopub.execute_input":"2021-09-14T01:16:45.452779Z","iopub.status.idle":"2021-09-14T01:18:03.366685Z","shell.execute_reply.started":"2021-09-14T01:16:45.452751Z","shell.execute_reply":"2021-09-14T01:18:03.365656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Vamos ver como foi o treino?\n\nfig, ax = plt.subplots(1,2, figsize=(16,8))\nax[0].plot(history.history['loss'], color='b', label=\"Training loss\")\nax[0].plot(history.history['val_loss'], color='r', label=\"validation loss\",axes =ax[0])\nlegend = ax[0].legend(loc='best', shadow=True)\n\nax[1].plot(history.history['accuracy'], color='b', label=\"Training accuracy\")\nax[1].plot(history.history['val_accuracy'], color='r',label=\"Validation accuracy\")\nlegend = ax[1].legend(loc='best', shadow=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-14T01:06:51.354482Z","iopub.execute_input":"2021-09-14T01:06:51.354846Z","iopub.status.idle":"2021-09-14T01:06:51.694025Z","shell.execute_reply.started":"2021-09-14T01:06:51.354816Z","shell.execute_reply":"2021-09-14T01:06:51.69305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Resultado do melhor modelo","metadata":{}},{"cell_type":"code","source":"MODEL_ID = \"model.bruno3\"\n\n# Carrega o melhor modelo\nmodel = load_model(MODEL_ID)\n\n# Avalia o melhor modelo\nscore = model.evaluate(x_val, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])\nprint('Test ROC AUC:', score[2])","metadata":{"execution":{"iopub.status.busy":"2021-09-14T01:23:39.789736Z","iopub.execute_input":"2021-09-14T01:23:39.790043Z","iopub.status.idle":"2021-09-14T01:23:41.075722Z","shell.execute_reply.started":"2021-09-14T01:23:39.790016Z","shell.execute_reply":"2021-09-14T01:23:41.07467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_ID = \"model.bruno10\"\n\n# Carrega o melhor modelo\nmodel = load_model(MODEL_ID)\n\n# Avalia o melhor modelo\nscore = model.evaluate(x_val, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])\nprint('Test ROC AUC:', score[2])","metadata":{"execution":{"iopub.status.busy":"2021-09-18T12:58:41.659169Z","iopub.execute_input":"2021-09-18T12:58:41.659517Z","iopub.status.idle":"2021-09-18T12:58:43.303553Z","shell.execute_reply.started":"2021-09-18T12:58:41.659485Z","shell.execute_reply":"2021-09-18T12:58:43.302471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-09-18T12:58:43.307210Z","iopub.execute_input":"2021-09-18T12:58:43.307522Z","iopub.status.idle":"2021-09-18T12:58:43.320216Z","shell.execute_reply.started":"2021-09-18T12:58:43.307475Z","shell.execute_reply":"2021-09-18T12:58:43.317902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"O melhor resultado com MLP sobre o dataset original, foi:\n```\nTest loss: 0.1159505842040692\nTest accuracy: 0.9640476\nTest roc auc: 0.9978843\n```\n\nJá o melhor resuiltado com CNN sobre o dataset original, foi:\n```\nTest loss: 0.032700711501474\nTest accuracy: 0.9909524\nTest ROC AUC: 0.9991975\n```","metadata":{}},{"cell_type":"markdown","source":"## Matriz de Confusão","metadata":{}},{"cell_type":"code","source":"import itertools\n#Plot the confusion matrix. Set Normalize = True/False\ndef plot_confusion_matrix(cm, classes, normalize=True, title='Confusion matrix', cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.figure(figsize=(10,10))\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        cm = np.around(cm, decimals=2)\n        cm[np.isnan(cm)] = 0.0\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')","metadata":{"execution":{"iopub.status.busy":"2021-09-18T12:58:43.322220Z","iopub.execute_input":"2021-09-18T12:58:43.322885Z","iopub.status.idle":"2021-09-18T12:58:43.338793Z","shell.execute_reply.started":"2021-09-18T12:58:43.322836Z","shell.execute_reply":"2021-09-18T12:58:43.336502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\n# Classificando toda base de teste\ny_pred = model.predict_classes(x_val)\n# voltando pro formato de classes\ny_test_c = np.argmax(y_val, axis=1)\ntarget_names = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']\n\n#Confution Matrix\ncm = confusion_matrix(y_test_c, y_pred)\nplot_confusion_matrix(cm, target_names, normalize=False, title='Confusion Matrix')\n\nprint('Classification Report')\nprint(classification_report(y_test_c, y_pred, target_names=target_names))","metadata":{"execution":{"iopub.status.busy":"2021-09-18T12:58:43.344752Z","iopub.execute_input":"2021-09-18T12:58:43.345395Z","iopub.status.idle":"2021-09-18T12:58:44.413049Z","shell.execute_reply.started":"2021-09-18T12:58:43.345346Z","shell.execute_reply":"2021-09-18T12:58:44.411930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Teste com Ruído","metadata":{}},{"cell_type":"code","source":"#introduzindo ruido\nimport numpy as np\nmean = 0.\nstddev = 0.2\nnoise = np.random.normal(mean, stddev, (4200, 28, 28,1))\nx_te = x_val + noise\nx_te = np.clip(x_te, 0., 1.)\n\nplt.imshow(x_te.reshape(4200, 28,28)[0], cmap=plt.cm.binary)\nplt.show()\n\n# Testa\nscore = model.evaluate(x_te, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])\nprint('Test ROC AUC:', score[2])","metadata":{"execution":{"iopub.status.busy":"2021-09-18T12:58:44.416060Z","iopub.execute_input":"2021-09-18T12:58:44.416513Z","iopub.status.idle":"2021-09-18T12:58:45.880190Z","shell.execute_reply.started":"2021-09-18T12:58:44.416465Z","shell.execute_reply":"2021-09-18T12:58:45.877716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"O melhor resultado com MLP sobre o dataset com ruído, foi:\n```\nTest loss: 0.9104807124819074\nTest accuracy: 0.7469048\nTest roc auc: 0.7469048\n```\n\nAgora com CNN sobre o dataset com ruído, foi:\n```\nTest loss: 0.041150492092794075\nTest accuracy: 0.98785716\nTest ROC AUC: 0.9991571\n```","metadata":{}},{"cell_type":"markdown","source":"# Visualizando o modelo","metadata":{}},{"cell_type":"code","source":"!pip3 install keras-visualizer","metadata":{"execution":{"iopub.status.busy":"2021-09-13T23:49:36.765716Z","iopub.execute_input":"2021-09-13T23:49:36.76606Z","iopub.status.idle":"2021-09-13T23:51:31.735104Z","shell.execute_reply.started":"2021-09-13T23:49:36.766033Z","shell.execute_reply":"2021-09-13T23:51:31.734249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras_visualizer import visualizer \n\nvisualizer(model, format='png', view=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-13T23:51:53.958693Z","iopub.execute_input":"2021-09-13T23:51:53.959052Z","iopub.status.idle":"2021-09-13T23:51:54.402692Z","shell.execute_reply.started":"2021-09-13T23:51:53.959018Z","shell.execute_reply":"2021-09-13T23:51:54.40114Z"},"trusted":true},"execution_count":null,"outputs":[]}]}