{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom keras.datasets import mnist\nfrom keras.utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import Conv2D, MaxPooling2D,AveragePooling2D,Average, Flatten, Dense, Dropout, Activation , Concatenate, Input , BatchNormalization\nfrom keras.optimizers import SGD\nfrom keras.utils import plot_model\nfrom matplotlib import pyplot as plt\nfrom keras.callbacks import EarlyStopping\nfrom keras.callbacks import ReduceLROnPlateau\nfrom keras import Model\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.utils import class_weight\nfrom collections import Counter","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# load data\ndf_train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndf_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# split our data into features & target\ntrainX = df_train.drop('label', axis=1).values\ntrainy = df_train['label'].values.reshape(-1,1)\n\ntestX = df_test.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainX[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainy[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"testX[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# summarize loaded dataset\nprint('Train: X=%s, y=%s' % (trainX.shape, trainy.shape))\nprint('Test: X=%s, y=%s' % (testX.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot first few images\nfor i in range(9):\n    img = trainX[i].reshape(28,28)\n    # define subplot\n    plt.subplot(330 + 1 + i)\n    # plot raw pixel data\n    plt.imshow(img)\n    \n# show the figure\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot first few images\nfor i in range(9):\n    img = trainX[i].reshape(28,28)\n    # define subplot\n    plt.subplot(330 + 1 + i)\n    # plot raw pixel data\n    plt.imshow(img,cmap=plt.get_cmap('gray') )\n    \n# show the figure\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# reshape dataset to have a single channel\ntrainX = trainX.reshape((trainX.shape[0], 28, 28, 1))\ntestX = testX.reshape((testX.shape[0], 28, 28, 1))\n# one hot encode target values\ntrainy = to_categorical(trainy)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# convert from integers to floats\ntrainX = trainX.astype('float32')\ntestX = testX.astype('float32')\n# normalize to range 0-1\ntrainX = trainX / 255.0\ntestX = testX / 255.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(trainX.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen = ImageDataGenerator(\n        validation_split = 0.25,\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.1, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        horizontal_flip=False,  # randomly flip images\n        vertical_flip=False)  # randomly flip images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 128\nTraining_data = datagen.flow(trainX,\n                             y=trainy,\n                            batch_size = batch_size,\n                            subset = 'training')\n\nValidation_data = datagen.flow(trainX,\n                             y=trainy,\n                            batch_size = batch_size,\n                            subset = 'validation')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# label_list = np.array([])\n\n# for i ,(data,label) in enumerate(Training_data):\n#     label_number = np.argmax(label, axis=1)\n#     label_list = np.concatenate((label_list, label_number), axis=None)   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class_weights = class_weight.compute_class_weight('balanced',np.unique(labels_np),label_list) \n# print(class_weights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"es = EarlyStopping(monitor='val_loss',\n                   mode='auto',\n                   verbose=1,\n                   patience=20)\n\nlrr= ReduceLROnPlateau(monitor='val_loss',\n                       factor=0.1, \n                       patience=5, \n                       min_lr=1e-9)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_model = Input((trainX.shape[1],trainX.shape[2],trainX.shape[3]))\n\n\nmodel1 = Conv2D(32,(5,5), activation='relu')(input_model)\nmodel1 = BatchNormalization()(model1)\nmodel1 = Conv2D(64,(5,5), activation='relu', padding='same')(model1)\nmodel1 = BatchNormalization()(model1)\nmodel1 = MaxPooling2D((2, 2))(model1)\nmodel1 = Conv2D(64,(3,3), activation='relu' ,padding='same')(model1)\nmodel1 = BatchNormalization()(model1)\nmodel1 = Conv2D(128,(3,3), activation='relu' ,padding='valid')(model1)\nmodel1 = BatchNormalization()(model1)\nmodel1 = AveragePooling2D((2, 2))(model1)\nmodel1 = Flatten()(model1)\nmodel1 = Dense(units = 512, activation = 'relu')(model1)\nmodel1 = Dropout(rate = 0.4)(model1)\nmodel1 = BatchNormalization()(model1)\nmodel1 = Dense(units = 20, activation = 'relu')(model1)\nmodel1 = BatchNormalization()(model1)\nmodel1 = Dense(units = 15, activation = 'relu')(model1)\nmodel1 = BatchNormalization()(model1)\noutput1 = Dense(units = 10, activation = 'softmax')(model1)\n#########################################################                          \nmodel2 = Conv2D(32,(4,4), activation='relu')(input_model)  \nmodel2 = BatchNormalization()(model2)\nmodel2 = Conv2D(32,(4,4), activation='relu', padding='same')(model2)\nmodel2 = BatchNormalization()(model2)\nmodel2 = MaxPooling2D((2, 2))(model2)\nmodel2 = Conv2D(32,(3,3), activation='relu', padding='same')(model2) \nmodel2 = BatchNormalization()(model2)\nmodel2 = Conv2D(32,(3,3), activation='relu', padding='same')(model2) \nmodel2 = BatchNormalization()(model2)\nmodel2 = Conv2D(64,(2,2), activation='relu' ,padding='same')(model2)\nmodel2 = BatchNormalization()(model2)\nmodel2 = Conv2D(128,(2,2), activation='relu' ,padding='valid')(model2)\nmodel2 = BatchNormalization()(model2)\nmodel2 = AveragePooling2D((2, 2))(model2)\nmodel2 = Flatten()(model2)\nmodel2 = Dense(units = 1024, activation = 'relu')(model2)\nmodel2 = Dropout(rate = 0.5)(model2)\nmodel2 = BatchNormalization()(model2)\nmodel2 = Dense(units = 64, activation = 'relu')(model2)\nmodel2 = BatchNormalization()(model2)\noutput2 = Dense(units = 10, activation = 'softmax')(model2)\n########################################################\nmodel3 = Conv2D(32,(3,3), activation='relu')(input_model)  \nmodel3 = BatchNormalization()(model3)\nmodel3 = Conv2D(32,(3,3), activation='relu', padding='same')(model3)\nmodel3 = BatchNormalization()(model3)\nmodel3 = MaxPooling2D((2, 2))(model3)\nmodel3 = Conv2D(32,(3,3), activation='relu', padding='same')(model3) \nmodel3 = BatchNormalization()(model3)\nmodel3 = Conv2D(64,(3,3), activation='relu', padding='same')(model3)\nmodel3 = BatchNormalization()(model3)\nmodel3 = Conv2D(64,(2,2), activation='relu' ,padding='valid')(model3)\nmodel3 = BatchNormalization()(model3)\nmodel3 = Conv2D(128,(2,2), activation='relu' ,padding='valid')(model3)\nmodel3 = BatchNormalization()(model3)\nmodel3 = MaxPooling2D((2, 2))(model3)\nmodel3 = Flatten()(model3)\nmodel3 = Dense(units = 512, activation = 'relu')(model3)\nmodel3 = Dropout(rate = 0.2)(model3)\nmodel3 = BatchNormalization()(model3)\nmodel3 = Dense(units = 64, activation = 'relu')(model3)\nmodel3 = BatchNormalization()(model3)\noutput3 = Dense(units = 10, activation = 'linear')(model3)\n########################################################\noutput = Average()([output1, output2 , output3])\noutput = Dense(units = 10, activation = 'softmax')(output)\n\nmodel = Model(inputs= [input_model], outputs=[output])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_model(model, show_shapes=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sgd = SGD(lr=0.1, momentum=0.99)\nmodel.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit_generator(Training_data, \n                              epochs= 400,\n                              validation_data= Validation_data,\n                              # class_weight=class_weights,\n                              verbose=1,\n                              callbacks=[es , lrr])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save_weights(\"MNIST_weights.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_loss = history.history['val_loss']\nloss = history.history['loss']\n\nplt.plot(val_loss)\nplt.plot(loss)\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend(['Val error','Train error'], loc='upper right')\nplt.savefig('plot_error.png')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_accuracy = history.history['val_accuracy']\naccuracy = history.history['accuracy']\n\nplt.plot(val_accuracy)\nplt.plot(accuracy)\nplt.xlabel('Epochs')\nplt.ylabel('accuracy')\nplt.legend(['Val accuracy','Train accuracy'], loc='upper right')\nplt.savefig( 'plot_accuracy.png')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = model.predict(testX)\npred = pd.DataFrame(pred)\npred['Label'] = pred.idxmax(axis=1)\npred.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred['index'] = list(range(1,len(pred)+1))\npred.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pred[['index','Label']]\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.rename(columns={'index':'ImageId'},inplace = True)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}