{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introdução","metadata":{}},{"cell_type":"markdown","source":"**Resumo:**\n\nO exemplo aqui desenvolvido tem como objetivo apresentar conceitos iniciais de implementação de redes neurais com python e tensorflow/keras. Esse modelo apresenta um modelo MLP básico que pode ser expandido mudando o número de neurônios e camadas. Em adaptações mais avançadas, pode-se estudar possibilidade de otimização de hyperparâmetros e outras técnincas como aumento de dados.\n\n**Não é objetivo nosso desenvolver e otimizar o modelo de classificação**. O exemplo tem objetivo meramente didático.\n\n---\n\n**Para saber mais:**\n* [Palestras e cursos do Ocean](http://www.oceanbrasil.com/)\n* François Chollet. Deep Learning with Python. Manning Publications, 2017.\n* Ian Goodfellow and Yoshua Bengio and Aaron Courville. [Deep Learning](https://www.deeplearningbook.org/). MIT Press, 2016.","metadata":{}},{"cell_type":"markdown","source":"# Bibliotecas e Dados","metadata":{}},{"cell_type":"code","source":"# Bibliotecas necessárias\n# Manipulação de dados\nimport pandas as pd\n# Redes Neurais\nfrom tensorflow import keras\nfrom tensorflow.keras.datasets import mnist\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout\nfrom tensorflow.keras.optimizers import RMSprop\n\nfrom sklearn.pipeline import Pipeline\n# Plot\nimport matplotlib.pyplot as plt\n%matplotlib inline\n# Avaliação\nfrom sklearn.metrics import classification_report, confusion_matrix","metadata":{"ExecuteTime":{"end_time":"2019-01-16T20:39:41.500258Z","start_time":"2019-01-16T20:39:39.261856Z"},"execution":{"iopub.status.busy":"2021-10-24T18:39:53.335157Z","iopub.execute_input":"2021-10-24T18:39:53.335620Z","iopub.status.idle":"2021-10-24T18:39:59.043445Z","shell.execute_reply.started":"2021-10-24T18:39:53.335589Z","shell.execute_reply":"2021-10-24T18:39:59.042406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lendo o dataset Kaggle\ntrain = pd.read_csv(\"../input/digit-recognizer/train.csv\")\n\n#Alternativa ler do próprio keras\n#(x_train, y_train), (x_test, y_test) = mnist.load_data()\n","metadata":{"execution":{"iopub.status.busy":"2021-10-24T18:39:59.047608Z","iopub.execute_input":"2021-10-24T18:39:59.048259Z","iopub.status.idle":"2021-10-24T18:40:02.907528Z","shell.execute_reply.started":"2021-10-24T18:39:59.048212Z","shell.execute_reply":"2021-10-24T18:40:02.906715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Analisando o dataset\nprint(\"Quantidade de elementos de treino: {}\". format(len(train)))\nprint(train.head())","metadata":{"execution":{"iopub.status.busy":"2021-10-24T18:40:02.909067Z","iopub.execute_input":"2021-10-24T18:40:02.909402Z","iopub.status.idle":"2021-10-24T18:40:02.925511Z","shell.execute_reply.started":"2021-10-24T18:40:02.909373Z","shell.execute_reply":"2021-10-24T18:40:02.924417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Separando x_train e y_train\nY = train[\"label\"]\nX = train.drop(labels = [\"label\"],axis = 1)\nprint(X.head())\n# Em formato numpy array de imagens 28 x 28\n#x = X.values.reshape(-1,28,28,1)\n#print(x[0])","metadata":{"execution":{"iopub.status.busy":"2021-10-24T18:40:02.927229Z","iopub.execute_input":"2021-10-24T18:40:02.927876Z","iopub.status.idle":"2021-10-24T18:40:03.053860Z","shell.execute_reply.started":"2021-10-24T18:40:02.927821Z","shell.execute_reply":"2021-10-24T18:40:03.052561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(Y)","metadata":{"execution":{"iopub.status.busy":"2021-10-24T18:40:03.058517Z","iopub.execute_input":"2021-10-24T18:40:03.058891Z","iopub.status.idle":"2021-10-24T18:40:03.068593Z","shell.execute_reply.started":"2021-10-24T18:40:03.058860Z","shell.execute_reply":"2021-10-24T18:40:03.067394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Numtendi nada!\n# Bora ver com matplotlib\nplt.imshow(X.values[100].reshape(28,28), cmap=plt.cm.binary)\nplt.show()\nprint('Label: {}'.format(Y[100]))","metadata":{"ExecuteTime":{"end_time":"2019-01-08T20:49:47.144492Z","start_time":"2019-01-08T20:49:47.016025Z"},"execution":{"iopub.status.busy":"2021-10-24T18:40:03.072027Z","iopub.execute_input":"2021-10-24T18:40:03.072503Z","iopub.status.idle":"2021-10-24T18:40:03.269685Z","shell.execute_reply.started":"2021-10-24T18:40:03.072463Z","shell.execute_reply":"2021-10-24T18:40:03.268664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Transformando a imagem 2d em um numpy array (imagem 28*28 = 784 pixels)\nx = X.values.reshape(42000, 784)\n\n#Normalizando para valores entre 0 e 1\nx = x.astype('float32')\nx /= 255\n\nprint(x[0])","metadata":{"ExecuteTime":{"end_time":"2019-01-08T20:49:57.425537Z","start_time":"2019-01-08T20:49:57.272377Z"},"execution":{"iopub.status.busy":"2021-10-24T18:40:03.271895Z","iopub.execute_input":"2021-10-24T18:40:03.272225Z","iopub.status.idle":"2021-10-24T18:40:03.373019Z","shell.execute_reply.started":"2021-10-24T18:40:03.272190Z","shell.execute_reply":"2021-10-24T18:40:03.372177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Vamos ajustar o formato da saida\nnum_classes = 10\n\n# Convertendo para um vetor de saida com 10 dimensoes\n# ex. 8 => [0,0,0,0,0,0,0,0,1,0]\ny = keras.utils.to_categorical(Y, num_classes)\nprint(y[0])","metadata":{"ExecuteTime":{"end_time":"2019-01-08T20:50:48.007428Z","start_time":"2019-01-08T20:50:47.986527Z"},"execution":{"iopub.status.busy":"2021-10-24T18:40:03.374458Z","iopub.execute_input":"2021-10-24T18:40:03.374735Z","iopub.status.idle":"2021-10-24T18:40:03.381338Z","shell.execute_reply.started":"2021-10-24T18:40:03.374707Z","shell.execute_reply":"2021-10-24T18:40:03.380332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Separando uma parte para treino (90%) e outra para validação (10%)\nfrom sklearn.model_selection import train_test_split\nx_train, x_val, y_train, y_val = train_test_split(x, y, test_size = 0.1, random_state=9)\nprint('Qtde de treino: {}'.format(len(x_train)))\nprint('Qtde de validação: {}'.format(len(x_val)))","metadata":{"execution":{"iopub.status.busy":"2021-10-24T18:40:03.382632Z","iopub.execute_input":"2021-10-24T18:40:03.382942Z","iopub.status.idle":"2021-10-24T18:40:03.772917Z","shell.execute_reply.started":"2021-10-24T18:40:03.382915Z","shell.execute_reply":"2021-10-24T18:40:03.772068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Criando e treinando o Modelo","metadata":{}},{"cell_type":"code","source":"# Criando o modelo Sequential\n# Sequential: Modelo Keras de ir adicionando camadas (como um lego)\n# Dense: Camada onde todas as entradas estão conectadas em cada neurônio (totalmente conectada)\n# Dropout: Camada usa durante treino que descarta aleatoriamente um percentual de conexões (reduz overfitting)\n\nmodel = Sequential()\n# Camada com 30 neurônios\nmodel.add(Dense(30, activation='relu', input_shape=(784,)))\n# Dropout de 20%\nmodel.add(Dropout(0.2))\n# Camada de 20 neurônios\nmodel.add(Dense(20, activation='relu'))\n# Dropout de 20%\nmodel.add(Dropout(0.2))\n# Camda de classificação final, com 1 neurônio para cada classe de saída. Softmax divide a probabilidade de cada classe.\nmodel.add(Dense(num_classes, activation='softmax'))\n\nmodel.summary()","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:38:00.233865Z","start_time":"2019-01-08T21:38:00.15079Z"},"scrolled":true,"execution":{"iopub.status.busy":"2021-10-24T18:40:03.774540Z","iopub.execute_input":"2021-10-24T18:40:03.774868Z","iopub.status.idle":"2021-10-24T18:40:03.929891Z","shell.execute_reply.started":"2021-10-24T18:40:03.774838Z","shell.execute_reply":"2021-10-24T18:40:03.929048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compila o modelo\nmodel.compile(loss='categorical_crossentropy',\n              optimizer=RMSprop(),\n              metrics=['accuracy'])","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:33:17.184644Z","start_time":"2019-01-08T21:33:17.129141Z"},"execution":{"iopub.status.busy":"2021-10-24T18:40:03.932233Z","iopub.execute_input":"2021-10-24T18:40:03.932679Z","iopub.status.idle":"2021-10-24T18:40:04.099310Z","shell.execute_reply.started":"2021-10-24T18:40:03.932636Z","shell.execute_reply":"2021-10-24T18:40:04.098436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Treina com os parte dos dados\nbatch_size = 32\nepochs = 30\nhistory = model.fit(x_train, y_train,\n                    batch_size=batch_size,\n                    epochs=epochs,\n                    verbose=1,\n                    validation_data=(x_val, y_val))","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:36:15.501168Z","start_time":"2019-01-08T21:33:19.322895Z"},"execution":{"iopub.status.busy":"2021-10-24T18:40:04.100614Z","iopub.execute_input":"2021-10-24T18:40:04.100892Z","iopub.status.idle":"2021-10-24T18:41:08.703442Z","shell.execute_reply.started":"2021-10-24T18:40:04.100865Z","shell.execute_reply":"2021-10-24T18:41:08.702507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Vamos ver como foi o treino?\n\nfig, ax = plt.subplots(1,2, figsize=(16,8))\nax[0].plot(history.history['loss'], color='b', label=\"Training loss\")\nax[0].plot(history.history['val_loss'], color='r', label=\"validation loss\",axes =ax[0])\nlegend = ax[0].legend(loc='best', shadow=True)\n\nax[1].plot(history.history['accuracy'], color='b', label=\"Training accuracy\")\nax[1].plot(history.history['val_accuracy'], color='r',label=\"Validation accuracy\")\nlegend = ax[1].legend(loc='best', shadow=True)","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:36:20.243921Z","start_time":"2019-01-08T21:36:20.064201Z"},"execution":{"iopub.status.busy":"2021-10-24T18:41:08.704694Z","iopub.execute_input":"2021-10-24T18:41:08.704982Z","iopub.status.idle":"2021-10-24T18:41:09.040764Z","shell.execute_reply.started":"2021-10-24T18:41:08.704955Z","shell.execute_reply":"2021-10-24T18:41:09.039830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Testa\nscore = model.evaluate(x_val, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:36:39.779842Z","start_time":"2019-01-08T21:36:38.946173Z"},"execution":{"iopub.status.busy":"2021-10-24T18:41:09.041997Z","iopub.execute_input":"2021-10-24T18:41:09.042343Z","iopub.status.idle":"2021-10-24T18:41:09.210016Z","shell.execute_reply.started":"2021-10-24T18:41:09.042312Z","shell.execute_reply":"2021-10-24T18:41:09.209234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y[10]","metadata":{"execution":{"iopub.status.busy":"2021-10-24T18:41:09.211039Z","iopub.execute_input":"2021-10-24T18:41:09.211476Z","iopub.status.idle":"2021-10-24T18:41:09.217174Z","shell.execute_reply.started":"2021-10-24T18:41:09.211444Z","shell.execute_reply":"2021-10-24T18:41:09.216259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Testando uma entrada qualquer\nprint(y_train[10])\nprint(model.predict(x_train[10].reshape((1,784))))\nprint(model.predict_classes(x_train[10].reshape((1,784))))","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:06:04.363715Z","start_time":"2019-01-08T21:06:04.27849Z"},"execution":{"iopub.status.busy":"2021-10-24T18:41:09.218325Z","iopub.execute_input":"2021-10-24T18:41:09.218835Z","iopub.status.idle":"2021-10-24T18:41:09.313966Z","shell.execute_reply.started":"2021-10-24T18:41:09.218792Z","shell.execute_reply":"2021-10-24T18:41:09.313015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Avaliando o Modelo","metadata":{}},{"cell_type":"code","source":"import itertools\n#Plot the confusion matrix. Set Normalize = True/False\ndef plot_confusion_matrix(cm, classes, normalize=True, title='Confusion matrix', cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.figure(figsize=(10,10))\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        cm = np.around(cm, decimals=2)\n        cm[np.isnan(cm)] = 0.0\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')","metadata":{"execution":{"iopub.status.busy":"2021-10-24T18:41:09.315290Z","iopub.execute_input":"2021-10-24T18:41:09.315581Z","iopub.status.idle":"2021-10-24T18:41:09.330259Z","shell.execute_reply.started":"2021-10-24T18:41:09.315554Z","shell.execute_reply":"2021-10-24T18:41:09.329077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Vendo alguns reports\n# Usando sklearn\nimport numpy as np\n\n# Classificando toda base de teste\ny_pred = model.predict_classes(x_val)\n# voltando pro formato de classes\ny_test_c = np.argmax(y_val, axis=1)\ntarget_names = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']\n\n#Confution Matrix\ncm = confusion_matrix(y_test_c, y_pred)\nplot_confusion_matrix(cm, target_names, normalize=False, title='Confusion Matrix')\n\nprint('Classification Report')\nprint(classification_report(y_test_c, y_pred, target_names=target_names))","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:24:57.7654Z","start_time":"2019-01-08T21:24:57.220867Z"},"execution":{"iopub.status.busy":"2021-10-24T18:41:09.331598Z","iopub.execute_input":"2021-10-24T18:41:09.331906Z","iopub.status.idle":"2021-10-24T18:41:10.084670Z","shell.execute_reply.started":"2021-10-24T18:41:09.331878Z","shell.execute_reply":"2021-10-24T18:41:10.083712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Experimental**","metadata":{}},{"cell_type":"markdown","source":"## Common Functions","metadata":{}},{"cell_type":"code","source":"def checkTrain(history, title):\n    fig, ax = plt.subplots(1,2, figsize=(16,8))\n    ax[0].plot(history.history['loss'], color='b', label=\"Training loss\")\n    ax[0].plot(history.history['val_loss'], color='r', label=\"validation loss\",axes =ax[0])\n    legend = ax[0].legend(loc='best', shadow=True)\n    fig.suptitle(title, fontsize=20)   \n\n    ax[1].plot(history.history['accuracy'], color='b', label=\"Training accuracy\")\n    ax[1].plot(history.history['val_accuracy'], color='r',label=\"Validation accuracy\")\n    legend = ax[1].legend(loc='best', shadow=True)\n\n# Controe a Rede Neural\ndef construct(layers, dropout, batch_size, epochs):\n    \n    title = 'Rede neural com '+str(len(layers))+' camadas: '+str(layers)+', Dropout de '+str(dropout)+', BatchSize: '+str(batch_size)+', Epochs: '+str(epochs)\n    model = Sequential()\n    \n    # Layers  \n    for n in layers:\n        model.add(Dense(n, activation='relu', input_shape=(784,)))\n        model.add(Dropout(dropout))\n        \n    # Softmax\n    model.add(Dense(10, activation='softmax'))\n    \n    print('\\n\\n', title, '\\n####################################################################################################')\n    model.summary()\n        \n    # Compile\n    model.compile(loss='categorical_crossentropy',\n              optimizer=RMSprop(),\n              metrics=['accuracy'])\n    \n    # Treina com os parte dos dados    \n    history = model.fit(x_train, y_train,\n                        batch_size=batch_size,\n                        epochs=epochs,\n                        verbose=1,\n                        validation_data=(x_val, y_val))\n    \n    print('#####################################################################################################################\\n\\n')\n    \n    checkTrain(history, title)  \n    return model  \n\ndef initialize(nns):\n    nnList = []\n    for row in ns.iterrows():\n        n = row[1][0] # qte de Neuronios\n        d = row[1][1] # Dropout (%percentual)\n        b = row[1][2] # batch_size\n        e = row[1][3] # epoch\n        \n        # Constroe Redes Neurais\n        nnList.append(construct(n, d, b, e))\n    \n    return nnList\n    ","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:34:24.611027Z","iopub.execute_input":"2021-10-24T19:34:24.611507Z","iopub.status.idle":"2021-10-24T19:34:24.633248Z","shell.execute_reply.started":"2021-10-24T19:34:24.611469Z","shell.execute_reply":"2021-10-24T19:34:24.632155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Initialize","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\n'''\nConstrução de Redes Neurais\n @Parâmentros:\n     n: Lista de camadas com numeros respectios e definidos de neurônios\n     d: Percentual de dropout\n     b: Tamanho do Batch\n     e: Numero de épocas\n\n return: Lista de Models Sequencials\n'''\n\n\ndata = [\n    {'n': [60,20,20], 'd': 0.2, 'b': 32, 'e': 15},\n    {'n': [20,20], 'd': 0.1, 'b': 32, 'e': 10},\n    {'n': [100,20], 'd': 0.1, 'b': 64, 'e': 30},\n    {'n': [100,70,20], 'd': 0.2, 'b': 64, 'e': 60},\n    {'n': [150,70,20], 'd': 0.2, 'b': 32, 'e': 30},\n    {'n': [200,150,100], 'd': 0.2, 'b': 32, 'e': 30}\n    \n]    \nns = pd.DataFrame(data)\nmodels = initialize(ns)","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:43:58.542109Z","iopub.execute_input":"2021-10-24T19:43:58.542554Z","iopub.status.idle":"2021-10-24T19:52:40.672347Z","shell.execute_reply.started":"2021-10-24T19:43:58.542518Z","shell.execute_reply":"2021-10-24T19:52:40.671254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Avaliando Modelos","metadata":{}},{"cell_type":"markdown","source":"### Classification Report","metadata":{}},{"cell_type":"code","source":"for model in models:\n    y_pred = model.predict_classes(x_val)\n    # voltando pro formato de classes\n    y_test_c = np.argmax(y_val, axis=1)\n    target_names = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']\n    \n    print('Classification Report - ' + str(model.layers[0].output.shape[1]))\n    print(classification_report(y_test_c, y_pred, target_names=target_names))\n    print('##################################################################\\n')\n","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:53:28.597356Z","iopub.execute_input":"2021-10-24T19:53:28.597721Z","iopub.status.idle":"2021-10-24T19:53:29.955104Z","shell.execute_reply.started":"2021-10-24T19:53:28.597690Z","shell.execute_reply":"2021-10-24T19:53:29.953164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Confusion Matrix","metadata":{}},{"cell_type":"code","source":"for model in models:\n    y_pred = model.predict_classes(x_val)\n    # voltando pro formato de classes\n    y_test_c = np.argmax(y_val, axis=1)\n    target_names = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']\n\n    # Confution Matrix\n    cm = confusion_matrix(y_test_c, y_pred)\n    plot_confusion_matrix(cm, target_names, normalize=False, title='Confusion Matrix - ' +str(model.layers[0].output.shape[1]))\n","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:53:35.867432Z","iopub.execute_input":"2021-10-24T19:53:35.867997Z","iopub.status.idle":"2021-10-24T19:53:40.356029Z","shell.execute_reply.started":"2021-10-24T19:53:35.867934Z","shell.execute_reply":"2021-10-24T19:53:40.354878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}