{"cells":[{"metadata":{"_uuid":"8c6c5c00845162d19b630edbbc34269cfe420ecc"},"cell_type":"markdown","source":"# Digit Recognizer\n\nTarget: Take an image of a handwritten single digit, and determine what that digit is.\n\nSource: https://www.kaggle.com/c/digit-recognizer"},{"metadata":{"trusted":true,"_uuid":"2dc4cb3acdea9893449c53f8edef4469cff60b56"},"cell_type":"code","source":"# data manipulation\nimport numpy as np\nimport pandas as pd\nimport random\n\n# high-level neural networks API - running on top of TensorFlow\nimport keras\n# Sequential is a linear stack of layers\nfrom keras.models import Sequential\n# Dense, Flatten - type of layers, Dropout - tool, which decrease chance of overfitting\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\nfrom keras import backend as K\n\n# data visualisation, live training loss plot\nimport matplotlib.pyplot as plt\n\nimport time\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d29da23df818e7745768936653df29c5ecaa1be0"},"cell_type":"code","source":"# load dataset\ntrain = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b3e6f6e42cac286c258740e2d3b6ca63135434d"},"cell_type":"code","source":"y_train = train[\"label\"]\nX_train = train.drop(labels = [\"label\"],axis = 1) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2f2bd17c99dddbfae818077a9288846f24d2bbe"},"cell_type":"code","source":"# shape of data\nX_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"661eed25656acfe5116bd004c6778345237bbb98"},"cell_type":"code","source":"def vis_rand_dig():\n    # size of pixcel\n    plt.figure(figsize = (10, 10))\n    # print random digit\n    rand_indexes = np.random.randint(0, X_train.shape[0], 16)\n    for index,im_index in enumerate(rand_indexes):\n        plt.subplot(4, 4, index+1)\n        plt.imshow(X_train.values[im_index].reshape(28,28), cmap = 'ocean', interpolation = 'none')\n        plt.title('Class %d' % y_train[im_index])\n    plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00cdae9fa02334fc6923929761d4b5a98d1dfd8f"},"cell_type":"code","source":"vis_rand_dig()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ebb266970fd6e91f2858e9cc7e69a2fe21a805a7"},"cell_type":"markdown","source":"# CNN\n\nConvolutional neural network expects additional parameter - number of channels. In computer science we have two type of standards. We may add channel at the first or last element. It is dependent on backend. I use TensorFlow, so I add channel at last."},{"metadata":{"trusted":true,"_uuid":"020b69af0b1b40f5da1857d999e89e1f2f7b48bb"},"cell_type":"code","source":"def prep_data(X_train, y_train, test):\n   \n    X_train = X_train.astype('float32') / 255\n    test = test.astype('float32')/255\n    X_train = X_train.values.reshape(-1,28,28,1)\n    test = test.values.reshape(-1,28,28,1)\n    y_train = keras.utils.np_utils.to_categorical(y_train)\n    classes = y_train.shape[1]\n    X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size = 0.2, random_state = int(time.time()))\n    \n    return X_train, y_train, X_test, y_test, classes, test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76ac9cd2d9e63500e3d34324890021f55d976555"},"cell_type":"code","source":"X_train, y_train, X_test, y_test, out_neurons, test = prep_data(X_train, y_train, test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa962e58f044307a2786ce0a3c6e4149db703c87"},"cell_type":"code","source":"def cnn():\n    cnn = Sequential([\n        Conv2D(32, kernel_size = (3, 3), padding = 'same', activation = 'relu', input_shape = (28,28,1)),\n        Conv2D(32, kernel_size = (3, 3), activation = 'relu', padding = 'same'),\n        MaxPool2D(pool_size = (2, 2)),\n        Dropout(0.25),\n        \n        Conv2D(64, kernel_size = (3, 3), activation = 'relu', padding = 'same'),\n        Conv2D(64, kernel_size = (3, 3), activation = 'relu', padding = 'same'),\n        MaxPool2D(pool_size = (2, 2)),\n        Dropout(0.25),\n        \n        Conv2D(128, kernel_size = (3, 3), activation = 'relu', padding = 'same'),\n        Conv2D(128, kernel_size = (3, 3), activation = 'relu', padding = 'same'),\n        MaxPool2D(pool_size = (2, 2)),\n        Dropout(0.25),\n        \n        Flatten(),\n        \n        Dense(512, activation = 'relu'),\n        Dropout(0.5),\n        Dense(256, activation = 'relu'),\n        Dropout(0.5),\n        Dense(out_neurons, activation = 'softmax')\n    ])\n    return cnn\n\nmodel = cnn()\nmodel.summary()\nmodel.compile(loss = 'categorical_crossentropy', optimizer = 'Adam' , metrics = ['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad17eae7537fe8657be0c2adfe7638c1039dda82"},"cell_type":"code","source":"model.fit(X_train, y_train,\n          batch_size = 512,\n          epochs = 11,\n          validation_data = (X_test, y_test),\n          verbose = 0);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bef1b63d5de51d3ec28dadb76fcdc66dc2f64211"},"cell_type":"code","source":"result = model.evaluate(X_test, y_test, verbose = 0)\nprint('Accuracy: ', result[1])\nprint('Error: %.2f%%' % (100- result[1]*100))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dc8a8a8708446b52d8046a38045cde937f95d6f7"},"cell_type":"markdown","source":"# Where we have mistake?"},{"metadata":{"trusted":true,"_uuid":"360bfa61533e9f7328a901ca34e2cbb59b965781"},"cell_type":"code","source":"y_pred = model.predict(test, verbose=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51ba15b14f0e183215d4d509228b167761532572"},"cell_type":"code","source":"def error_predict(y_test, y_pred):\n    for idx, (a, b) in enumerate(zip(y_test, y_pred)):\n        if np.argmax(a) == np.argmax(b): continue\n        yield idx, np.argmax(a), tuple(np.argsort(b)[-2:])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54a1238795fc336d60b3e78fd6778f458c40fe9c"},"cell_type":"code","source":"def display_errors():\n    random_errors = random.sample(list(error_predict(y_test, y_pred)), 12)\n\n    plt.figure(figsize=(10, 10))\n    X_test_plot = X_test.reshape(X_test.shape[0], X_test.shape[1], X_test.shape[2] )\n\n    for index, (im_index, y_test_val, (y_pred_2, y_pred_1)) in enumerate(random_errors):\n            plt.subplot(4,4,index+1)\n            plt.imshow(X_test_plot[im_index], cmap='ocean', interpolation='none')\n            plt.title('True value: {0}\\nFirst predicted: {1}\\nSecond predicted: {2}'.format(y_test_val, y_pred_1, y_pred_2))\n            plt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e4905aab3b77caab646d8654c75917720d09aa7"},"cell_type":"code","source":"display_errors()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1c5b38c9bdf8227d650a78ee5cdb07554946170b"},"cell_type":"markdown","source":"In my opiniom model make mistake in very difficult cases.\n\n# Conclusions\n\n- The best score: 0.9952(accuracy), 0,48%(error);\n- The best architecture of CNN: \n        Conv2D(32, kernel_size = (3, 3), padding = 'same', activation = 'relu',\n                input_shape = in_dim),\n        Conv2D(32, kernel_size = (3, 3), activation = 'relu', padding = 'same'),\n        MaxPool2D(pool_size = (2, 2)),\n        Dropout(0.25),\n        \n        Conv2D(64, kernel_size = (3, 3), activation = 'relu', padding = 'same'),\n        Conv2D(64, kernel_size = (3, 3), activation = 'relu', padding = 'same'),\n        MaxPool2D(pool_size = (2, 2)),\n        Dropout(0.25),\n        \n        Conv2D(128, kernel_size = (3, 3), activation = 'relu', padding = 'same'),\n        Conv2D(128, kernel_size = (3, 3), activation = 'relu', padding = 'same'),\n        MaxPool2D(pool_size = (2, 2)),\n        Dropout(0.25),\n        \n        Flatten(),\n        \n        Dense(512, activation = 'relu'),\n        Dropout(0.5),\n        Dense(256, activation = 'relu'),\n        Dropout(0.5),\n        Dense(out_dim, activation = 'softmax')\n        \n- Parameters: 1,010,666;\n- Time: 16 min 47 s;\n- Hardware: CPU - Ryzen 2700X  (I know that GPU is better for deep learning, but I didn't have opportunity to work on GPU);\n- Batch size: 512;\n- Epochs: 11;\n\nI tried several configuration. The results varied between 0,48% - 0,84% (error). Generally I changed number of layers, number of kernels, size of batch_size and number of epochs. \nI noticed, that: \n- More layer increased computing time and growing number of parameters;\n- More batch_size affect the reduced computing time;\n- More expanded section of MLP (number of layers and number of neurones) influences on more parameters;\n- More CNN layers = less parameters;\n- More numbers of epochs affect on growing compute time;\n- Worse score after used BatchNormalization;\n- CNN vs MLP for digit recognition? Decisively CNN for digit recognizer!"},{"metadata":{"trusted":true,"_uuid":"cbff4572f1225208936470b5d48a19a569a8f150"},"cell_type":"code","source":"solution = np.argmax(y_pred,axis = 1)\nsolution = pd.Series(solution, name=\"Label\").astype(int)\nsubmission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),solution],axis = 1)\nsubmission.to_csv(\"mnist_cnn.csv\",index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}