{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"trains=pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntests=pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\nsampleSubs=pd.read_csv(\"/kaggle/input/digit-recognizer/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trains.describe().T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trains.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print (trains.dtypes[:5])# all int64, otherwise do train = train.astype('int64')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Extract the train data and target**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_y=trains[\"label\"]\ntrain_X=trains.drop(\"label\",axis=1)\n# train_X=trains.iloc[:,1:]  # an alternatively method\ntest_X=tests\n\nprint (\"the shape of training data is:\", train_X.shape)\nprint (\"the shape of target data is:\", train_y.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data exploration ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\nsns.countplot(train_y)\n\n# present the counts of each digits.\ncounts=train_y.value_counts()\nprint (type(counts))\nprint (counts)\n\nmean_val=np.mean(counts)\nstd_val=np.std(counts,ddof=1)\nprint (\"the mean value of counts is:\",mean_val)\nprint (\"the std value of counts is:\",std_val)\n\n# 68% - 95% - 99% rule, For an approximately normal data set, \n#the values within one standard deviation of the mean account for about 68% of the set; \n#while within two standard deviations account for about 95%; \n# https://en.wikipedia.org/wiki/68%E2%80%9395%E2%80%9399.7_rule\n\n# if the data is skewed then we won't be able to use accurace as its results will be misleading and we may use F-beta score instead.\nprint (mean_val* (0.6827 / 2))\nif std_val > mean_val* (0.6827 / 2):\n    print(\"The standard deviation is high\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# check null data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_missData(df):\n    # check the missing data of dataFrame.\n    # output them in the form of dataFrame\n    # sum(): count the number of missing data in each column\n    # isnull(): return a dataFrame with True, and False\n    \n    miss_tot=df.isnull().sum().sort_values(ascending=False)\n    counts_all=df.isnull().count() # count all the elements, including the missing elements\n    miss_per=((df.isnull().sum())*100/counts_all).sort_values(ascending=False)\n    miss_all=pd.concat([miss_tot,miss_per],axis=1,keys=['TotalNum','TotalPerc'])\n    return miss_all\ncheck_missData(train_X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show image, the shape of image --> (28,28)\n%matplotlib inline\nimport matplotlib.pyplot as plt\n# index=10\n# image=train_X.iloc[index].values\n# image=image.reshape((28,28)) \n# plt.imshow(image,cmap=plt.cm.get_cmap('gray'), interpolation=\"nearest\")\n# # plt.show()\n\n\n# convert train dataset to (num_images, img_rows, img_cols) format in order to plot it\nnumtrain=train_X.shape[0]\ndim=28\nxtrain_vis = train_X.values.reshape(numtrain, dim, dim)\n\n# subplot(2,3,3) = subplot(233)\n# a grid of 3x3 is created, then plots are inserted in some of these slots\nfor i in range(0,9): # how many imgs will show from the 3x3 grid\n    plt.subplot(330 + (i+1)) # open next subplot\n    plt.imshow(xtrain_vis[i], cmap=plt.get_cmap('gray'))\n    plt.title(train_y[i]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nimport tensorflow as tf\nfrom tensorflow import keras\n\nfrom tensorflow.keras.models import Sequential,Model\nfrom tensorflow.keras.layers import Conv2D, Dense,Dropout,MaxPooling2D,Flatten\nfrom tensorflow.keras.layers import Input, ZeroPadding2D,BatchNormalization\nfrom tensorflow.keras.utils import to_categorical ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Normalize data and reshape","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X=train_X/255-0.5\ntest_X=test_X/255-0.5\ntrain_X=train_X.values\ntest_X=test_X.values\ntrain_y=train_y.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X=np.reshape(train_X,(42000,28,28,-1))  # or train_X.values.reshape(-1,28,28,1) # -1 indicate not care  of the number of axis=0 location\ntest_X=np.reshape(test_X,(28000,28,28,-1))\n\nprint (\"the shape of train_X is reshaped: \", train_X.shape)\nprint (\"the shape of test_X is reshaped:\",test_X.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_y=to_categorical(train_y)  # one-hot \nprint (train_y.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# split training data ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# fix random seed for reproducibility\nseed = 2\nnp.random.seed(seed)\n\n# percentage of xtrain which will be x_validation\nsplit_pct = 0.1\n\n# Split the train and the validation set\nx_train, x_val, y_train, y_val = train_test_split(train_X,\n                                              train_y, \n                                              test_size=split_pct,\n                                              random_state=seed,\n                                              shuffle=True,\n                                              stratify=train_y\n                                             )\n\nprint(x_train.shape, y_train.shape, x_val.shape, y_val.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Construct the nn network ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# five nets are used for accuracy prediction\nnets=5\nmodel=[0]*nets # define a zero list with length of nets\n\nfor i in range(nets):\n    \n    model[i]=Sequential()\n\n    model[i].add(Conv2D(32,(5,5),strides=(1,1),activation=\"relu\",input_shape=(28,28,1),name='conv0')) # shape=(height,width,depth)  -->24*24*32\n    model[i].add(BatchNormalization())\n    model[i].add(MaxPooling2D((2,2),name=\"max_pool_1\")) # --> 12*12*32\n    model[i].add(Conv2D(32,(5,5),strides=(1,1),activation=\"relu\",padding=\"same\",name='conv1'))  #--> 12*12*64\n    model[i].add(BatchNormalization())\n    model[i].add(Dropout(0.4))\n    \n    model[i].add(Conv2D(64,(3,3),strides=(1,1),activation='relu',name='conv2')) #-->10*10*128\n    model[i].add(BatchNormalization())\n    model[i].add(MaxPooling2D((2,2),name=\"max_pool_2\")) #--> 5*5*128\n    model[i].add(Conv2D(64,(3,3),strides=(1,1),activation=\"relu\",padding=\"same\",name='conv4')) #-->5*5*192\n    model[i].add(BatchNormalization())\n    model[i].add(Dropout(0.4))\n    \n    model[i].add(Conv2D(128,(4,4),strides=(1,1),activation='relu',name='conv5'))  #-->2*2*320\n    model[i].add(BatchNormalization())\n    model[i].add(Flatten())\n    model[i].add(Dropout(0.4))\n    model[i].add(Dense(128,activation='relu',name='fc_1'))\n    model[i].add(Dense(64,activation='relu',name='fc_2'))\n    model[i].add(Dense(10,activation='softmax',name=\"fc_3\"))\n    model[i].compile(optimizer='adam',loss=\"categorical_crossentropy\",metrics=[\"acc\"])  # be careful, either \"acc\" or \"accuracy\" in metrics due to the new version of keras 2.3.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\ndatagen = ImageDataGenerator(\n          featurewise_center=False,            # set input mean to 0 over the dataset\n          samplewise_center=False,             # set each sample mean to 0\n          featurewise_std_normalization=False, # divide inputs by std of the dataset\n          samplewise_std_normalization=False,  # divide each input by its std\n          zca_whitening=False,                 # apply ZCA whitening\n          rotation_range=10,                   # randomly rotate images in the range (degrees, 0 to 180)\n          zoom_range = 0.1,                    # Randomly zoom image \n          width_shift_range=0.1,               # randomly shift images horizontally (fraction of total width)\n          height_shift_range=0.1,              # randomly shift images vertically (fraction of total height)\n          horizontal_flip=False,               # randomly flip images\n          vertical_flip=False)                 # randomly flip images\n\n# datagen.fit(x_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# there are several ways for the learning_rate_decay. here an example of exponetial decay is presented \nplt.figure()\nx=np.linspace(0,10,100)\nalpha=0.001*np.power(0.85,x)\nplt.plot(x,alpha)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.callbacks import ReduceLROnPlateau, LearningRateScheduler\n\n# noted:  do not mixure the keras and tf.keras, otherwise, attribution error would occur. \n# lrs = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x) #  expoential decay. \n# callbacks=[annealer], \n\ndef lr_scheduler(epoch, lr):\n    # exponential decay\n    # variation epoch, lr could be input automatically. \n    decay_rate = 0.95\n    decay_step = 10\n    if epoch % decay_step == 0 and epoch:\n        return lr * pow(decay_rate, np.floor(epoch / decay_step))\n    return lr\n\ncallbacks = [LearningRateScheduler(lr_scheduler, verbose=1)]\n\n# train the network\nhistory = [0] * nets\nepochs = 50\n\nfor j in range(nets):\n    \n    print(\"CNN \",j+1)\n    X_train2, X_val2, Y_train2, Y_val2 = train_test_split(x_train, y_train, test_size = 0.1)\n    history[j] = model[j].fit_generator(datagen.flow(X_train2,Y_train2, batch_size=64),\n                                        epochs = epochs, \n                                        steps_per_epoch = X_train2.shape[0]//64,  \n                                        validation_data = (X_val2,Y_val2), \n                                        callbacks=callbacks,\n                                        verbose=1)\n    print(\"CNN {0:d}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}\".format(\n        j+1,epochs,max(history[j].history['acc']),max(history[j].history['val_acc']) ))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# plot the loss function of model fitting","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(2,1) # creat a 2 rows, 1 column plot zones. \n\nax[0].plot(history[3].history['loss'],color='b',label=\"Trainning loss\")\nax[0].plot(history[4].history['val_loss'], color='r', label=\"Validation loss\",axes =ax[0])\n\nax[0].grid(color='black', linestyle='-', linewidth=0.25)\nlegend = ax[0].legend(loc='best', shadow=True)\n\nax[1].plot(history[4].history['acc'], color='b', label=\"Training accuracy\")\nax[1].plot(history[4].history['val_acc'], color='r',label=\"Validation accuracy\")\nax[1].grid(color='black', linestyle='-', linewidth=0.25)\nlegend = ax[1].legend(loc='best', shadow=True)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Ensemble results \n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"results = np.zeros( (test_X.shape[0],10) )  #--> shape (28000,10)\n\nfor j in range(nets):\n    # obtain the maximum probality by adding five cnn. (28000,10)\n    results = results + model[j].predict(test_X)  # return a numpy ndarray.\n    \nresults = np.argmax(results,axis = 1)\nresults = pd.Series(results,name=\"Label\")\n\nsubmission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\nsubmission.to_csv(\"predictions_again.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}