{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this notebook i'll try to build a simple Digit Recognizer model with the help of neural network since i'm a beginner","metadata":{}},{"cell_type":"code","source":"#importing necessary libraries\nimport numpy as np \nimport pandas as pd \n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom keras.models import Sequential\nfrom keras.layers import Input,Conv2D,Activation,Dense,Lambda,Flatten,Embedding,PReLU,BatchNormalization,Dropout,MaxPooling2D,GlobalMaxPooling2D,GlobalAveragePooling2D\nfrom sklearn.model_selection import train_test_split\nfrom keras.models import Model\nfrom keras import  backend as K\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import EarlyStopping\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.losses import categorical_crossentropy, binary_crossentropy, mean_squared_error,sparse_categorical_crossentropy\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau,LearningRateScheduler, TensorBoard, EarlyStopping, ModelCheckpoint","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:18:00.511990Z","iopub.execute_input":"2021-11-02T16:18:00.512333Z","iopub.status.idle":"2021-11-02T16:18:00.529356Z","shell.execute_reply.started":"2021-11-02T16:18:00.512300Z","shell.execute_reply":"2021-11-02T16:18:00.528415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Importing the dataset","metadata":{}},{"cell_type":"code","source":"train=pd.read_csv('../input/digit-recognizer/train.csv')\nprint(train.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:18:00.531507Z","iopub.execute_input":"2021-11-02T16:18:00.532232Z","iopub.status.idle":"2021-11-02T16:18:03.341774Z","shell.execute_reply.started":"2021-11-02T16:18:00.532186Z","shell.execute_reply":"2021-11-02T16:18:03.340753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test=pd.read_csv('../input/digit-recognizer/test.csv')\nprint(test.shape)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:18:03.343314Z","iopub.execute_input":"2021-11-02T16:18:03.344302Z","iopub.status.idle":"2021-11-02T16:18:05.451090Z","shell.execute_reply.started":"2021-11-02T16:18:03.344246Z","shell.execute_reply":"2021-11-02T16:18:05.449888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=train.to_numpy()\ntest =test.to_numpy()","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:18:05.453665Z","iopub.execute_input":"2021-11-02T16:18:05.454315Z","iopub.status.idle":"2021-11-02T16:18:05.460295Z","shell.execute_reply.started":"2021-11-02T16:18:05.454253Z","shell.execute_reply":"2021-11-02T16:18:05.458988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"X_train = train[:,1:] # all pixel values\ny_train = train[:,0] # only labels i.e targets digits\nX_test  = test\n\n\nn_rows = X_train.shape[0]\nn_cols = X_train.shape[1]\nmean = 0.3\nstddev = 0.1\nnoise = np.random.normal(mean, stddev, (n_rows, n_cols))\n# creating the noisy test data by adding X_test with noise\n#X_train = X_train+ noise","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:18:05.462138Z","iopub.execute_input":"2021-11-02T16:18:05.462957Z","iopub.status.idle":"2021-11-02T16:18:06.895815Z","shell.execute_reply.started":"2021-11-02T16:18:05.462903Z","shell.execute_reply":"2021-11-02T16:18:06.894779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#expand 1 more dimention as 1 for colour channel gray\nX_train = X_train.reshape(X_train.shape[0], 28, 28,1)\nX_test = X_test.reshape(X_test.shape[0], 28, 28,1)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:18:06.897786Z","iopub.execute_input":"2021-11-02T16:18:06.898313Z","iopub.status.idle":"2021-11-02T16:18:06.904737Z","shell.execute_reply.started":"2021-11-02T16:18:06.898268Z","shell.execute_reply":"2021-11-02T16:18:06.903790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Standardization","metadata":{}},{"cell_type":"code","source":"# one hot encoding the labels\n\nfrom keras.utils.np_utils import to_categorical\ny_train= to_categorical(y_train)\nnum_classes = y_train.shape[1]\nnum_classes","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:18:06.906216Z","iopub.execute_input":"2021-11-02T16:18:06.906893Z","iopub.status.idle":"2021-11-02T16:18:06.920651Z","shell.execute_reply.started":"2021-11-02T16:18:06.906845Z","shell.execute_reply":"2021-11-02T16:18:06.919252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Designing","metadata":{}},{"cell_type":"code","source":"from keras.preprocessing import image\ngen = image.ImageDataGenerator()\n# fix random seed for reproducibility\nseed = 42\nnp.random.seed(seed)\n\nfrom sklearn.model_selection import train_test_split\nX = X_train\ny = y_train\nX_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.10, random_state=42)\nbatches = gen.flow(X_train, y_train, batch_size=64)\nval_batches=gen.flow(X_val, y_val, batch_size=64)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:18:06.923034Z","iopub.execute_input":"2021-11-02T16:18:06.923548Z","iopub.status.idle":"2021-11-02T16:18:07.536555Z","shell.execute_reply.started":"2021-11-02T16:18:06.923427Z","shell.execute_reply":"2021-11-02T16:18:07.535484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Simple Model","metadata":{}},{"cell_type":"code","source":"inputs=Input((28,28,1))\nx=Conv2D(32,(3,3),kernel_initializer='he_uniform')(inputs)\nx=BatchNormalization()(x)\nx=PReLU()(x)\n#x=Dropout(0.5)(x)\n\nx=Conv2D(32,(3,3),kernel_initializer='he_uniform')(x)\nx=BatchNormalization()(x)\nx=PReLU()(x)\n#x=Dropout(0.5)(x)\n\nx=Conv2D(64,(3,3),kernel_initializer='he_uniform')(x)\nx=BatchNormalization()(x)\nx=PReLU()(x)\n#x=Dropout(0.5)(x)\n\nx=Conv2D(64,(3,3),kernel_initializer='he_uniform')(x)\nx=BatchNormalization()(x)\nx=PReLU()(x)\nx=MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n#x=Dropout(0.5)(x)\n\nx=Conv2D(128,(3,3),kernel_initializer='he_uniform')(x)\nx=BatchNormalization()(x)\nx=PReLU()(x)\nx=MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n#x=Dropout(0.5)(x)\n\nx=Conv2D(128,(3,3),kernel_initializer='he_uniform')(x)\nx=BatchNormalization()(x)\nx=PReLU()(x)\n#x=Dropout(0.5)(x)\n\n#x=Flatten()(x)\nx=GlobalAveragePooling2D()(x)\nx=Dense(64)(x)\nout1=PReLU(name=\"out1\")(x)\n\nout2=Dense(10,activation=\"softmax\")(out1)\n\nmodel=Model(inputs,out2)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:18:07.540275Z","iopub.execute_input":"2021-11-02T16:18:07.540763Z","iopub.status.idle":"2021-11-02T16:18:07.769146Z","shell.execute_reply.started":"2021-11-02T16:18:07.540684Z","shell.execute_reply":"2021-11-02T16:18:07.768067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(loss=categorical_crossentropy,\n              optimizer=Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-7),\n              metrics=['accuracy'])\n\nlr_reducer = ReduceLROnPlateau('val_accuracy', factor=0.08, patience=3, verbose=1, mode='auto')\ntensorboard = TensorBoard(log_dir='./logs')\nearly_stopper = EarlyStopping(monitor='val_accuracy', min_delta=0, patience=8, verbose=1, mode='auto')\n\nmodel.fit(X_train,y_train,batch_size=32,epochs=100,verbose=1,validation_data=(X_val,y_val),\n         shuffle=True,\n          callbacks=[lr_reducer, tensorboard, early_stopper] )\n                    ","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:18:07.771031Z","iopub.execute_input":"2021-11-02T16:18:07.771355Z","iopub.status.idle":"2021-11-02T16:23:02.179002Z","shell.execute_reply.started":"2021-11-02T16:18:07.771308Z","shell.execute_reply":"2021-11-02T16:23:02.177760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for layer in model.layers:\n#    layer.trainable = False\n#new_model= Sequential()\n#for layer in model.layers[:-1]: #just exclude the last two layers from base_model\n#    new_model.add(layer)\n#new_model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:23:02.180598Z","iopub.execute_input":"2021-11-02T16:23:02.181230Z","iopub.status.idle":"2021-11-02T16:23:02.188615Z","shell.execute_reply.started":"2021-11-02T16:23:02.181172Z","shell.execute_reply":"2021-11-02T16:23:02.187427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#x3_train=new_model.predict(X_train)\n#x3_test =new_model.predict(X_test)\n#x3_val =new_model.predict(X_val)\n#y3_train=np.argmax(y_train, axis=1)\n#y3_val=np.argmax(y_val, axis=1)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:23:02.190688Z","iopub.execute_input":"2021-11-02T16:23:02.192010Z","iopub.status.idle":"2021-11-02T16:23:02.198700Z","shell.execute_reply.started":"2021-11-02T16:23:02.191964Z","shell.execute_reply":"2021-11-02T16:23:02.197467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from sklearn.metrics import accuracy_score,hinge_loss\n#from sklearn.ensemble import RandomForestClassifier\n#clf=RandomForestClassifier()\n#hystory=clf.fit(x3_train, y3_train)\n##y_pred = clf.predict(x3_test)\n#y_pred = clf.predict(x3_val)\n#print(\"Accuracy:\",accuracy_score(y3_val, y_pred))","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:23:02.200736Z","iopub.execute_input":"2021-11-02T16:23:02.201215Z","iopub.status.idle":"2021-11-02T16:23:02.210628Z","shell.execute_reply.started":"2021-11-02T16:23:02.201130Z","shell.execute_reply":"2021-11-02T16:23:02.209519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict(X_test,batch_size=64)\nimage_id = range(1, predictions.shape[0] + 1)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:23:02.212690Z","iopub.execute_input":"2021-11-02T16:23:02.213141Z","iopub.status.idle":"2021-11-02T16:23:34.416850Z","shell.execute_reply.started":"2021-11-02T16:23:02.213095Z","shell.execute_reply":"2021-11-02T16:23:34.415786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = [np.argmax(i) for i in predictions]","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:23:34.418810Z","iopub.execute_input":"2021-11-02T16:23:34.419122Z","iopub.status.idle":"2021-11-02T16:23:34.513887Z","shell.execute_reply.started":"2021-11-02T16:23:34.419078Z","shell.execute_reply":"2021-11-02T16:23:34.512760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = pd.read_csv('../input/digit-recognizer/sample_submission.csv')\nsample.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:23:34.515468Z","iopub.execute_input":"2021-11-02T16:23:34.516073Z","iopub.status.idle":"2021-11-02T16:23:34.539758Z","shell.execute_reply.started":"2021-11-02T16:23:34.515985Z","shell.execute_reply":"2021-11-02T16:23:34.538788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({'ImageId': image_id, 'Label': pred})\nsubmission.to_csv('digit_recognizer_submission', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:23:34.541001Z","iopub.execute_input":"2021-11-02T16:23:34.541312Z","iopub.status.idle":"2021-11-02T16:23:34.620761Z","shell.execute_reply.started":"2021-11-02T16:23:34.541270Z","shell.execute_reply":"2021-11-02T16:23:34.619822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-02T16:23:34.621990Z","iopub.execute_input":"2021-11-02T16:23:34.622773Z","iopub.status.idle":"2021-11-02T16:23:34.632986Z","shell.execute_reply.started":"2021-11-02T16:23:34.622689Z","shell.execute_reply":"2021-11-02T16:23:34.631830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}}]}