{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Aluno: Daniel Benoliel Malcher\n\nMatrícula: 181080002\n\n\nAlterações feitas estão em anotações em markdown\n\nAlém das alterações diretas no código, retirei algumas células de markdown para facilitar leitura das alterações.\nPor trabalhar em PAIC utilizando CNNs, fiz algumas alterações que já sabia que iriam melhorar a acurácia mas também experimentei com alguns aspectos que ainda não tinha utilizado para o MNIST em si. ","metadata":{}},{"cell_type":"markdown","source":"# Bibliotecas e Dados","metadata":{}},{"cell_type":"code","source":"# Bibliotecas necessárias\n# Manipulação de dados\nimport pandas as pd\n# Redes Neurais\nfrom tensorflow import keras\nfrom tensorflow.keras.datasets import mnist\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, Conv2D, MaxPooling2D, Flatten\nfrom tensorflow.keras.optimizers import RMSprop\n# Plot\nimport matplotlib.pyplot as plt\n%matplotlib inline\n# Avaliação\nfrom sklearn.metrics import classification_report, confusion_matrix\n","metadata":{"ExecuteTime":{"end_time":"2019-01-16T20:39:41.500258Z","start_time":"2019-01-16T20:39:39.261856Z"},"execution":{"iopub.status.busy":"2022-03-11T03:43:13.936139Z","iopub.execute_input":"2022-03-11T03:43:13.936452Z","iopub.status.idle":"2022-03-11T03:43:19.168833Z","shell.execute_reply.started":"2022-03-11T03:43:13.936420Z","shell.execute_reply":"2022-03-11T03:43:19.167888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lendo o dataset Kaggle\ntrain = pd.read_csv(\"../input/digit-recognizer/train.csv\")\n\n#Alternativa ler do próprio keras\n#(x_train, y_train), (x_test, y_test) = mnist.load_data()\n","metadata":{"execution":{"iopub.status.busy":"2022-03-11T03:43:19.170503Z","iopub.execute_input":"2022-03-11T03:43:19.170761Z","iopub.status.idle":"2022-03-11T03:43:22.829050Z","shell.execute_reply.started":"2022-03-11T03:43:19.170735Z","shell.execute_reply":"2022-03-11T03:43:22.828237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Analisando o dataset\nprint(\"Quantidade de elementos de treino: {}\". format(len(train)))\nprint(train.head())","metadata":{"execution":{"iopub.status.busy":"2022-03-11T03:43:22.830442Z","iopub.execute_input":"2022-03-11T03:43:22.830818Z","iopub.status.idle":"2022-03-11T03:43:22.845772Z","shell.execute_reply.started":"2022-03-11T03:43:22.830778Z","shell.execute_reply":"2022-03-11T03:43:22.844353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Separando x_train e y_train\nY = train[\"label\"]\nX = train.drop(labels = [\"label\"],axis = 1)\nprint(X.head())\n# Em formato numpy array de imagens 28 x 28\n#x = X.values.reshape(-1,28,28,1)\n#print(x[0])","metadata":{"execution":{"iopub.status.busy":"2022-03-11T03:43:22.846932Z","iopub.execute_input":"2022-03-11T03:43:22.847231Z","iopub.status.idle":"2022-03-11T03:43:22.909760Z","shell.execute_reply.started":"2022-03-11T03:43:22.847198Z","shell.execute_reply":"2022-03-11T03:43:22.908921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X.shape)","metadata":{"execution":{"iopub.status.busy":"2022-03-11T03:43:22.912145Z","iopub.execute_input":"2022-03-11T03:43:22.912368Z","iopub.status.idle":"2022-03-11T03:43:22.916575Z","shell.execute_reply.started":"2022-03-11T03:43:22.912344Z","shell.execute_reply":"2022-03-11T03:43:22.915689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Numtendi nada!\n# Bora ver com matplotlib\nplt.imshow(X.values[0].reshape(28,28), cmap=plt.cm.binary)\nplt.show()\nprint('Label: {}'.format(Y[0]))","metadata":{"ExecuteTime":{"end_time":"2019-01-08T20:49:47.144492Z","start_time":"2019-01-08T20:49:47.016025Z"},"execution":{"iopub.status.busy":"2022-03-11T03:43:22.920393Z","iopub.execute_input":"2022-03-11T03:43:22.920809Z","iopub.status.idle":"2022-03-11T03:43:23.082298Z","shell.execute_reply.started":"2022-03-11T03:43:22.920783Z","shell.execute_reply":"2022-03-11T03:43:23.081317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Transformando a imagem 2d em um numpy array (imagem 28*28)\nx = X.values.reshape(42000, 28, 28, 1)\n\n#Normalizando para valores entre 0 e 1\nx = x.astype('float32')\nx /= 255\n\n#print(x[0])","metadata":{"ExecuteTime":{"end_time":"2019-01-08T20:49:57.425537Z","start_time":"2019-01-08T20:49:57.272377Z"},"execution":{"iopub.status.busy":"2022-03-11T03:43:23.083379Z","iopub.execute_input":"2022-03-11T03:43:23.083684Z","iopub.status.idle":"2022-03-11T03:43:23.130997Z","shell.execute_reply.started":"2022-03-11T03:43:23.083659Z","shell.execute_reply":"2022-03-11T03:43:23.129743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Vamos ajustar o formato da saida\nnum_classes = 10\n\n# Convertendo para um vetor de saida com 10 dimensoes\n# ex. 8 => [0,0,0,0,0,0,0,0,1,0]\ny = keras.utils.to_categorical(Y, num_classes)\nprint(y[0])","metadata":{"ExecuteTime":{"end_time":"2019-01-08T20:50:48.007428Z","start_time":"2019-01-08T20:50:47.986527Z"},"execution":{"iopub.status.busy":"2022-03-11T03:43:23.132129Z","iopub.execute_input":"2022-03-11T03:43:23.132350Z","iopub.status.idle":"2022-03-11T03:43:23.140898Z","shell.execute_reply.started":"2022-03-11T03:43:23.132327Z","shell.execute_reply":"2022-03-11T03:43:23.139495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Separando uma parte para treino (90%) e outra para validação (10%)\nfrom sklearn.model_selection import train_test_split\nx_train, x_val, y_train, y_val = train_test_split(x, y, test_size = 0.1, random_state=5)\nprint('Qtde de treino: {}'.format(len(x_train)))\nprint('Qtde de validação: {}'.format(len(x_val)))","metadata":{"execution":{"iopub.status.busy":"2022-03-11T03:43:23.142152Z","iopub.execute_input":"2022-03-11T03:43:23.142516Z","iopub.status.idle":"2022-03-11T03:43:23.360542Z","shell.execute_reply.started":"2022-03-11T03:43:23.142478Z","shell.execute_reply":"2022-03-11T03:43:23.359826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Criando e treinando o Modelo","metadata":{}},{"cell_type":"markdown","source":"**Alteração:\nAdicionei uma segunda camada de MaxPooling2D após a segunda camada Conv2D.\nDropout foi alterado de 0.2 para 0.5.**\n\nEu tentei também alterar o kernel_size para 32 e 64 nas camadas Conv2D, no entanto a acurácia diminuiu.","metadata":{}},{"cell_type":"code","source":"# Criando o modelo Sequential\n# Sequential: Modelo Keras de ir adicionando camadas (como um lego)\n# Conv2D: Camada com kernels (filtros) que percorrem a imagem extraindo caracterísitcas (mapas de caracteísticas)\n# MaxPooling2D: Camada que reduz a dimensionalidade dos mapas de características 2D\n# Flatten: Camada que transforma um mapa de características 2D num vetor para classficador final\n# Dense: Camada onde todas as entradas estão conectadas em cada neurônio (totalmente conectada)\n# Dropout: Camada usa durante treino que descarta aleatoriamente um percentual de conexões (reduz overfitting)\n\nmodel = Sequential()\nmodel.add(Conv2D(20, kernel_size=(3, 3),\n                 activation='relu',\n                 input_shape=(28,28,1)))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Conv2D(50, kernel_size=(3,3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Flatten())\nmodel.add(Dense(500, activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(num_classes, activation='softmax'))\nmodel.summary()","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:38:00.233865Z","start_time":"2019-01-08T21:38:00.15079Z"},"scrolled":true,"execution":{"iopub.status.busy":"2022-03-11T03:43:23.361728Z","iopub.execute_input":"2022-03-11T03:43:23.361957Z","iopub.status.idle":"2022-03-11T03:43:23.533658Z","shell.execute_reply.started":"2022-03-11T03:43:23.361934Z","shell.execute_reply":"2022-03-11T03:43:23.531846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Alteração: Mudei o optimizer para adam**","metadata":{}},{"cell_type":"code","source":"# Compila o modelo\nmodel.compile(loss='categorical_crossentropy',\n              optimizer=\"adam\",\n              metrics=['accuracy'])","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:33:17.184644Z","start_time":"2019-01-08T21:33:17.129141Z"},"execution":{"iopub.status.busy":"2022-03-11T03:43:23.534681Z","iopub.execute_input":"2022-03-11T03:43:23.534948Z","iopub.status.idle":"2022-03-11T03:43:23.570803Z","shell.execute_reply.started":"2022-03-11T03:43:23.534922Z","shell.execute_reply":"2022-03-11T03:43:23.570147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Experimentei com algumas alterações no número de épocas e patience, mas o aumento de épocas efetivas de treino não resultava em uma alteração na acurácia quando a mesma estabilizava, apenas aumentava o tempo de treino.**","metadata":{}},{"cell_type":"markdown","source":"**Alteração: ModelCheckpoint e EarlyStopping: monitor = 'val_accuracy'**\n\nNotei que a maior acurácia de treino não era a mesma do encontrado por model.evaluate(), portanto alterei a variável que esses callbacks monitoram.","metadata":{}},{"cell_type":"code","source":"# Treina com os parte dos dados\nbatch_size = 32\nepochs = 20\n\n#Salvar o melhor modelo\ncallbacks_list = [\n    keras.callbacks.ModelCheckpoint(\n        filepath='model.h5',\n        monitor='val_accuracy', save_best_only=True, verbose=1),\n    keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=10,verbose=1)\n]\n\nhistory = model.fit(x_train, y_train,\n                    batch_size=batch_size,\n                    epochs=epochs,\n                    callbacks = callbacks_list,\n                    verbose=1,\n                    validation_data=(x_val, y_val))","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:36:15.501168Z","start_time":"2019-01-08T21:33:19.322895Z"},"execution":{"iopub.status.busy":"2022-03-11T03:50:35.329738Z","iopub.execute_input":"2022-03-11T03:50:35.330041Z","iopub.status.idle":"2022-03-11T03:54:28.491035Z","shell.execute_reply.started":"2022-03-11T03:50:35.330012Z","shell.execute_reply":"2022-03-11T03:54:28.489318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Vamos ver como foi o treino?\n\nfig, ax = plt.subplots(1,2, figsize=(16,8))\nax[0].plot(history.history['loss'], color='b', label=\"Training loss\")\nax[0].plot(history.history['val_loss'], color='r', label=\"validation loss\",axes =ax[0])\nlegend = ax[0].legend(loc='best', shadow=True)\n\nax[1].plot(history.history['accuracy'], color='b', label=\"Training accuracy\")\nax[1].plot(history.history['val_accuracy'], color='r',label=\"Validation accuracy\")\nlegend = ax[1].legend(loc='best', shadow=True)","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:36:20.243921Z","start_time":"2019-01-08T21:36:20.064201Z"},"execution":{"iopub.status.busy":"2022-03-11T03:54:51.476857Z","iopub.execute_input":"2022-03-11T03:54:51.477179Z","iopub.status.idle":"2022-03-11T03:54:51.744328Z","shell.execute_reply.started":"2022-03-11T03:54:51.477151Z","shell.execute_reply":"2022-03-11T03:54:51.743294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.models import load_model\n# Load the best saved model\nmodel = load_model('model.h5')","metadata":{"execution":{"iopub.status.busy":"2022-03-11T03:54:57.208068Z","iopub.execute_input":"2022-03-11T03:54:57.208373Z","iopub.status.idle":"2022-03-11T03:54:57.522210Z","shell.execute_reply.started":"2022-03-11T03:54:57.208346Z","shell.execute_reply":"2022-03-11T03:54:57.521568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Testa\nscore = model.evaluate(x_val, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:36:39.779842Z","start_time":"2019-01-08T21:36:38.946173Z"},"execution":{"iopub.status.busy":"2022-03-11T03:55:00.392661Z","iopub.execute_input":"2022-03-11T03:55:00.393104Z","iopub.status.idle":"2022-03-11T03:55:01.046450Z","shell.execute_reply.started":"2022-03-11T03:55:00.393075Z","shell.execute_reply":"2022-03-11T03:55:01.045497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Testando uma entrada qualquer\nprint(y_train[10])\nprint(model.predict(x_train[10].reshape((1,28,28,1))))\nprint(model.predict_classes(x_train[10].reshape((1,28,28,1))))","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:06:04.363715Z","start_time":"2019-01-08T21:06:04.27849Z"},"execution":{"iopub.status.busy":"2022-03-11T03:49:01.513365Z","iopub.execute_input":"2022-03-11T03:49:01.513607Z","iopub.status.idle":"2022-03-11T03:49:01.617685Z","shell.execute_reply.started":"2022-03-11T03:49:01.513584Z","shell.execute_reply":"2022-03-11T03:49:01.616946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Avaliando o Modelo","metadata":{}},{"cell_type":"code","source":"import itertools\n\n#Plot the confusion matrix. Set Normalize = True/False\ndef plot_confusion_matrix(cm, classes, normalize=True, title='Confusion matrix', cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.figure(figsize=(10,10))\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        cm = np.around(cm, decimals=2)\n        cm[np.isnan(cm)] = 0.0\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')","metadata":{"execution":{"iopub.status.busy":"2022-03-11T03:49:01.618615Z","iopub.execute_input":"2022-03-11T03:49:01.618874Z","iopub.status.idle":"2022-03-11T03:49:01.633752Z","shell.execute_reply.started":"2022-03-11T03:49:01.618846Z","shell.execute_reply":"2022-03-11T03:49:01.632924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Vendo alguns reports# Vendo alguns reports\n# Usando sklearn\nimport numpy as np\n\n# Classificando toda base de teste\ny_pred = model.predict_classes(x_val)\n# voltando pro formato de classes\ny_test_c = np.argmax(y_val, axis=1)\ntarget_names = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']\n\n#Confution Matrix\ncm = confusion_matrix(y_test_c, y_pred)\nplot_confusion_matrix(cm, target_names, normalize=False, title='Confusion Matrix')\n\nprint('Classification Report')\nprint(classification_report(y_test_c, y_pred, target_names=target_names))","metadata":{"ExecuteTime":{"end_time":"2019-01-08T21:24:57.7654Z","start_time":"2019-01-08T21:24:57.220867Z"},"execution":{"iopub.status.busy":"2022-03-11T03:49:01.634774Z","iopub.execute_input":"2022-03-11T03:49:01.635145Z","iopub.status.idle":"2022-03-11T03:49:02.676603Z","shell.execute_reply.started":"2022-03-11T03:49:01.635116Z","shell.execute_reply":"2022-03-11T03:49:02.675530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Gerando Saída","metadata":{}},{"cell_type":"code","source":"# Gerando saída para dataset de teste\n\n#Carrega dataset de teste\ntest = pd.read_csv(\"../input/digit-recognizer/test.csv\")\nprint(\"Qtde de testes: {}\".format(len(test)))\n# Bota no formato numpy e normaliza\nx_test = test.values.reshape(len(test),28,28,1)\nx_test = x_test.astype('float32')\nx_test /= 255\n\n# Faz classificação para dataset de teste\ny_pred = model.predict_classes(x_test)\n\n# Verficando algum exemplo\ni = 0\nplt.imshow(test.values[i].reshape(28,28), cmap=plt.cm.binary)\nplt.show()\nprint('Previsto: {}'.format(y_pred[i]))\n\n# Botando no formato de saída (competição Kaggle)\nresults = pd.Series(y_pred,name=\"Label\")\nsubmission = pd.concat([pd.Series(range(1,len(y_pred)+1),name = \"ImageId\"),results],axis = 1)\nprint(submission.head(10))\n#Salvando Arquivo\nsubmission.to_csv(\"mlp_mnist_v1.csv\",index=False)\nsubmission.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-03-11T03:49:02.677695Z","iopub.execute_input":"2022-03-11T03:49:02.677937Z","iopub.status.idle":"2022-03-11T03:49:08.642693Z","shell.execute_reply.started":"2022-03-11T03:49:02.677912Z","shell.execute_reply":"2022-03-11T03:49:08.641661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Teste Adicional: Com ruído","metadata":{}},{"cell_type":"code","source":"#introduzindo ruido\nimport numpy as np\nmean = 0.\nstddev = 0.2\nnoise = np.random.normal(mean, stddev, (4200, 28, 28,1))\nx_te = x_val + noise\nx_te = np.clip(x_te, 0., 1.)\n\nplt.imshow(x_te.reshape(4200, 28,28)[0], cmap=plt.cm.binary)\nplt.show()\n\n# Testa\nscore = model.evaluate(x_te, y_val, verbose=0)\nprint('Test loss:', score[0])\nprint('Test accuracy:', score[1])","metadata":{"execution":{"iopub.status.busy":"2022-03-11T03:49:08.643725Z","iopub.execute_input":"2022-03-11T03:49:08.644021Z","iopub.status.idle":"2022-03-11T03:49:09.538936Z","shell.execute_reply.started":"2022-03-11T03:49:08.643991Z","shell.execute_reply":"2022-03-11T03:49:09.537945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}