{"cells":[{"metadata":{},"cell_type":"markdown","source":"# MINST\n\nAuthored by Aczy156\n\nLearn from https://www.kaggle.com/yassineghouzam/introduction-to-cnn-keras-0-997-top-6"},{"metadata":{},"cell_type":"markdown","source":"## prework\n\n* load dependencies\n* load data\n* check data"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# necessary libuaray\n\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\nsns.set(style='white', context='notebook', palette='deep')\n%matplotlib inline\n\nnp.random.seed(2)\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nimport itertools\n\nfrom keras.utils.np_utils import to_categorical # attribute change => for one-hot encoding\n# label encoding ==> categorical varibles ==>进行替换，但是并没有增加新的\n# some other encoding(one-hot encoding, label encoding, 连续的合并)\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\n# change CNN structure\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.preprocessing import image\nfrom keras.optimizers import Adam\n\nfrom keras.optimizers import RMSprop\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load data\ntrain = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check data\nY_train = train['label']\n\n# drop 'label' column\nX_train = train.drop(labels=['label'], axis=1)\n\n# free space\ndel train\n\ng = sns.countplot(Y_train)\n\nY_train.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check null\nX_train.isnull().any().describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.isnull().any().describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data preprocessing\n\n* Normalization 因为是像素点，所以要特殊处理，把数据都搞成(0, 1)之间\n* Reshape 因为图像原因，要重塑图像，(height, width, channel维度=1) 维度为1的原因是他已经不是三原色了，只是单一的色调"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Normalization\nX_train = X_train / 255.0\ntest = test / 255.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reshape\nX_train = X_train.values.reshape(-1, 28, 28, 1)\ntest = test.values.reshape(-1, 28, 28, 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Feature engineering\n* label encoding\n* don't need to add any more feature"},{"metadata":{"trusted":true},"cell_type":"code","source":"# one-hot encoding\nY_train = to_categorical(Y_train, num_classes=10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model\n\n* split train and validate set\n* build model\n* train\n    * set optimizer\n* compile\n    * set annealer"},{"metadata":{"trusted":true},"cell_type":"code","source":"random_seed = 2\nX_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size=0.1, random_state=random_seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"g = plt.imshow(X_train[0][:,:,0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model\n\n\n# # CNN\n# model = Sequential()\n# model.add(Conv2D(filters=32, kernel_size=(5, 5), padding='Same', activation='relu', input_shape=(28, 28, 1)))\n# model.add(Conv2D(filters=32, kernel_size=(5, 5), padding='Same', activation='relu', input_shape=(28, 28, 1)))\n# model.add(MaxPool2D(pool_size=(2, 2)))\n# model.add(Dropout(0.25))\n# model.add(Conv2D(filters=64, kernel_size=(3, 3), padding='Same', activation='relu'))\n# model.add(Conv2D(filters=64, kernel_size=(3, 3), padding='Same', activation='relu'))\n# model.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2)))\n# model.add(Dropout(0.25))\n\n# model.add(Flatten())\n# model.add(Dense(256, activation='relu'))\n# model.add(Dropout(0.5))\n# model.add(Dense(10, activation='softmax'))\n\n\n# change CNN structure\n\nmodel = Sequential()\nmodel.add(Conv2D(filters=32, kernel_size=(3, 3), padding='Same', activation='relu', input_shape=(28, 28, 1)))\n# add two layers => avoid overfit\nmodel.add(BatchNormalization(axis=1)) \nmodel.add(Dropout(0.5))\nmodel.add(Conv2D(filters=32, kernel_size=(3, 3), padding='Same', activation='relu', input_shape=(28, 28, 1)))\nmodel.add(BatchNormalization(axis=1)) \n\nmodel.add(MaxPool2D(pool_size=(2, 2)))\n\nmodel.add(Conv2D(filters=64, kernel_size=(3, 3), padding='Same', activation='relu'))\nmodel.add(BatchNormalization(axis=1)) \nmodel.add(Dropout(0.5))\nmodel.add(Conv2D(filters=64, kernel_size=(3, 3), padding='Same', activation='relu'))\nmodel.add(MaxPool2D(pool_size=(2, 2)))\n\nmodel.add(Flatten())\nmodel.add(BatchNormalization()) \nmodel.add(Dropout(0.5))\nmodel.add(Dense(512, activation='relu'))\nmodel.add(BatchNormalization()) \nmodel.add(Dropout(0.5))\nmodel.add(Dense(10, activation='softmax'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# use keras method to generate more data\ngen = image.ImageDataGenerator(rotation_range=10, width_shift_range=0.10, shear_range=0.5, height_shift_range=0.10, zoom_range=0.10)\n# set other config => (compile)\nmodel.compile(Adam(), loss='categorical_crossentropy', metrics=['accuracy'])\n# optimizer and data batches\nmodel.optimizer.lr=0.001\nbatches = gen.flow(X_train, Y_train, batch_size=64)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# submit\npredictions = model.predict_classes(test, verbose=0)\nsubmissions=pd.DataFrame({'ImageId': list(range(1,len(predictions)+1)), 'Label': predictions})\nsubmissions.to_csv(\"changeCNN_all.csv\",index=False, header=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # model fit\n# history = model_fit_generator(generator=batches, steps_per_epoch=batches.n, epochs=15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # set optimizer and then compile\n# optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\n# model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # set rate annealer and evaluate\n# learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', patience=3, verbose=1, factor=0.5, min_lr=0.00001)\n# epochs = 1 # if epochs = 3 => overfit, lower score\n# batch_size = 86","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## data augmentation\n(资料扩充)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# # first: without data augmentation\n# history = model.fit(X_train, Y_train, batch_size, epochs=epochs, validation_data=(X_val, Y_val), verbose=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # then: submit and predict\n# results = model.predict(test)\n# results = np.argmax(results, axis=1)\n# results = pd.Series(results, name='Label')\n\n# # submit\n# submission = pd.concat([pd.Series(range(1, 28001), name='ImageId'), results], axis=1)\n\n# submission.to_csv('basicCNN_without_dataGen.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # after generate more data (=>由于当前的数据较少，会造成过拟合)\n# datagen = ImageDataGenerator(featurewise_center=False,\n#                              samplewise_center=False,\n#                              featurewise_std_normalization=False,\n#                              samplewise_std_normalization=False,\n#                              zca_whitening=False,\n#                              rotation_range=10,\n#                              zoom_range=0.1,\n#                              width_shift_range=0.1,\n#                              height_shift_range=0.1,\n#                              horizontal_flip=False,\n#                              vertical_flip=False\n#                             )\n# datagen.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # after gendata, fit\n# history = model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size), epochs=epochs, validation_data=(X_val, Y_val), verbose=2, steps_per_epoch=X_train.shape[0], callbacks=[learning_rate_reduction])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # then: submit and predict\n# results = model.predict(test)\n# results = np.argmax(results, axis=1)\n# results = pd.Series(results, name='Label')\n\n# # submit\n# submission = pd.concat([pd.Series(range(1, 28001), name='ImageId'), results], axis=1)\n\n# submission.to_csv('changeCNN_structure[add-BN].csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Evaluate model\n\n* training and validation curves\n* confusion matrix 混淆矩阵\n* display some error"},{"metadata":{"trusted":true},"cell_type":"code","source":"# # evaluate\n\n# fig, ax = plt.subplots(2,1)\n# ax[0].plot(history.history['loss'], color='b', label=\"Training loss\")\n# ax[0].plot(history.history['val_loss'], color='r', label=\"validation loss\",axes =ax[0])\n# legend = ax[0].legend(loc='best', shadow=True)\n\n# ax[1].plot(history.history['acc'], color='b', label=\"Training accuracy\")\n# ax[1].plot(history.history['val_acc'], color='r',label=\"Validation accuracy\")\n# legend = ax[1].legend(loc='best', shadow=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # confusion matrix\n\n# def plot_confusion_matrix(cm, classes,\n#                           normalize=False,\n#                           title='Confusion matrix',\n#                           cmap=plt.cm.Blues):\n#     \"\"\"\n#     This function prints and plots the confusion matrix.\n#     Normalization can be applied by setting `normalize=True`.\n#     \"\"\"\n#     plt.imshow(cm, interpolation='nearest', cmap=cmap)\n#     plt.title(title)\n#     plt.colorbar()\n#     tick_marks = np.arange(len(classes))\n#     plt.xticks(tick_marks, classes, rotation=45)\n#     plt.yticks(tick_marks, classes)\n\n#     if normalize:\n#         cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n#     thresh = cm.max() / 2.\n#     for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n#         plt.text(j, i, cm[i, j],\n#                  horizontalalignment=\"center\",\n#                  color=\"white\" if cm[i, j] > thresh else \"black\")\n\n#     plt.tight_layout()\n#     plt.ylabel('True label')\n#     plt.xlabel('Predicted label')\n\n# # Predict the values from the validation dataset\n# Y_pred = model.predict(X_val)\n# # Convert predictions classes to one hot vectors \n# Y_pred_classes = np.argmax(Y_pred,axis = 1) \n# # Convert validation observations to one hot vectors\n# Y_true = np.argmax(Y_val,axis = 1) \n# # compute the confusion matrix\n# confusion_mtx = confusion_matrix(Y_true, Y_pred_classes) \n# # plot the confusion matrix\n# plot_confusion_matrix(confusion_mtx, classes = range(10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # display some error\n# errors = (Y_pred_classes - Y_true != 0)\n\n# Y_pred_classes_errors = Y_pred_classes[errors]\n# Y_pred_errors = Y_pred[errors]\n# Y_true_errors = Y_true[errors]\n# X_val_errors = X_val[errors]\n\n# def display_errors(errors_index,img_errors,pred_errors, obs_errors):\n#     \"\"\" This function shows 6 images with their predicted and real labels\"\"\"\n#     n = 0\n#     nrows = 2\n#     ncols = 3\n#     fig, ax = plt.subplots(nrows,ncols,sharex=True,sharey=True)\n#     for row in range(nrows):\n#         for col in range(ncols):\n#             error = errors_index[n]\n#             ax[row,col].imshow((img_errors[error]).reshape((28,28)))\n#             ax[row,col].set_title(\"Predicted label :{}\\nTrue label :{}\".format(pred_errors[error],obs_errors[error]))\n#             n += 1\n\n# # Probabilities of the wrong predicted numbers\n# Y_pred_errors_prob = np.max(Y_pred_errors,axis = 1)\n\n# # Predicted probabilities of the true values in the error set\n# true_prob_errors = np.diagonal(np.take(Y_pred_errors, Y_true_errors, axis=1))\n\n# # Difference between the probability of the predicted label and the true label\n# delta_pred_true_errors = Y_pred_errors_prob - true_prob_errors\n\n# # Sorted list of the delta prob errors\n# sorted_dela_errors = np.argsort(delta_pred_true_errors)\n\n# # Top 6 errors \n# most_important_errors = sorted_dela_errors[-6:]\n\n# # Show the top 6 errors\n# display_errors(most_important_errors, X_val_errors, Y_pred_classes_errors, Y_true_errors)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}